Compare commits

..
1 Commits
Author SHA1 Message Date
death-claw 2220410f02 version bump 2024-12-14 17:56:07 +01:00
66 changed files with 632 additions and 718 deletions
+31 -51
View File
@@ -8,12 +8,12 @@ jobs:
build:
strategy:
matrix:
os: [ ubuntu-latest, windows-latest, macos-13, macos-latest ]
os: [ ubuntu-latest, windows-latest, macos-latest ]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- name: Set up jdk21
- name: Set up JDK 21
uses: actions/setup-java@v3
with:
java-version: '21'
@@ -21,29 +21,37 @@ jobs:
server-id: github # Value of the distributionManagement/repository/id field of the pom.xml
settings-path: ${{ github.workspace }} # location for the settings.xml file
- name: Build gui jar
run: |
mvn -N -q install --file pom.xml
mvn -B -q install --file vripper-core/pom.xml
mvn -B -q install --file vripper-gui/pom.xml
mv vripper-gui/target/vripper-gui-${{ github.event.release.tag_name }}-jar-with-dependencies.jar vripper-gui/target/vripper-noarch-gui-${{ github.event.release.tag_name }}.jar
- name: Build parent project
run: mvn -N -q install --file pom.xml
- name: Build Core Jar
run: mvn -B -q install --file vripper-core/pom.xml
- name: Build GUI Jar
run: mvn -B -q install --file vripper-gui/pom.xml
- name: Rename GUI Jar
run: mv vripper-gui/target/vripper-gui-${{ github.event.release.tag_name }}-jar-with-dependencies.jar vripper-gui/target/vripper-noarch-gui-${{ github.event.release.tag_name }}.jar
- if: matrix.os == 'ubuntu-latest'
name: Release gui Jar
name: Release GUI Jar
uses: softprops/action-gh-release@v1
with:
files: vripper-gui/target/vripper-noarch-gui-${{ github.event.release.tag_name }}.jar
# Start building WEB jar in ubuntu only
- if: matrix.os == 'ubuntu-latest'
name: Build web Jar
name: Build WEB Jar
run: |
mvn -B -q install --file vripper-web-ui/pom.xml
mvn -B -q install --file vripper-web/pom.xml
mv vripper-web/target/vripper-web-${{ github.event.release.tag_name }}.jar vripper-web/target/vripper-noarch-web-${{ github.event.release.tag_name }}.jar
- if: matrix.os == 'ubuntu-latest'
name: Release web Jar
name: Rename WEB Jar
run: mv vripper-web/target/vripper-web-${{ github.event.release.tag_name }}.jar vripper-web/target/vripper-noarch-web-${{ github.event.release.tag_name }}.jar
- if: matrix.os == 'ubuntu-latest'
name: Release WEB Jar
uses: softprops/action-gh-release@v1
with:
files: vripper-web/target/vripper-noarch-web-${{ github.event.release.tag_name }}.jar
@@ -54,7 +62,7 @@ jobs:
cp vripper-gui/target/vripper-noarch-gui-${{ github.event.release.tag_name }}.jar jpackage/jar/vripper-gui.jar
- if: matrix.os == 'ubuntu-latest'
name: Package for Linux
name: Package Linux
run: |
cd jpackage
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.png
@@ -64,7 +72,7 @@ jobs:
mv dist/vripper_${{ github.event.release.tag_name }}-1_amd64.deb dist/vripper-linux-${{ github.event.release.tag_name }}_amd64.deb
- if: matrix.os == 'windows-latest'
name: Package for Windows
name: Package Windows
run: |
cd jpackage
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.ico
@@ -75,24 +83,14 @@ jobs:
ren VRipper-${{ github.event.release.tag_name }}.exe vripper-windows-installer-${{ github.event.release.tag_name }}.exe
- if: matrix.os == 'macos-latest'
name: Package for macOS(arm64)
name: Package macOS
run: |
cd jpackage
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.icns
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type pkg
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type dmg
mv dist/VRipper-${{ github.event.release.tag_name }}.pkg dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.pkg
mv dist/VRipper-${{ github.event.release.tag_name }}.dmg dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.dmg
- if: matrix.os == 'macos-13'
name: Package for macOS(x86_64)
run: |
cd jpackage
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.icns
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type pkg
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type dmg
mv dist/VRipper-${{ github.event.release.tag_name }}.pkg dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.pkg
mv dist/VRipper-${{ github.event.release.tag_name }}.dmg dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.dmg
mv dist/VRipper-${{ github.event.release.tag_name }}.pkg dist/vripper-macos-${{ github.event.release.tag_name }}.pkg
mv dist/VRipper-${{ github.event.release.tag_name }}.dmg dist/vripper-macos-${{ github.event.release.tag_name }}.dmg
- if: matrix.os == 'ubuntu-latest'
name: Zip Ubuntu portable
@@ -113,22 +111,13 @@ jobs:
filename: 'vripper-windows-portable-${{ github.event.release.tag_name }}.zip'
- if: matrix.os == 'macos-latest'
name: Zip macOS(arm64) portable
name: Zip MacOS portable
uses: thedoctor0/zip-release@0.7.1
with:
type: 'zip'
directory: 'jpackage/dist'
path: 'VRipper.app'
filename: 'vripper-macos-portable-${{ github.event.release.tag_name }}.arm64.zip'
- if: matrix.os == 'macos-13'
name: Zip macOS(x86_64) portable
uses: thedoctor0/zip-release@0.7.1
with:
type: 'zip'
directory: 'jpackage/dist'
path: 'VRipper.app'
filename: 'vripper-macos-portable-${{ github.event.release.tag_name }}.x86_64.zip'
filename: 'vripper-macos-portable-${{ github.event.release.tag_name }}.zip'
- if: matrix.os == 'ubuntu-latest'
name: Release packages for Linux
@@ -149,19 +138,10 @@ jobs:
jpackage/dist/vripper-windows-portable-${{ github.event.release.tag_name }}.zip
- if: matrix.os == 'macos-latest'
name: Release packages for macOS(arm64)
name: Release packages for macOS
uses: softprops/action-gh-release@v1
with:
files: |
jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.pkg
jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.dmg
jpackage/dist/vripper-macos-portable-${{ github.event.release.tag_name }}.arm64.zip
- if: matrix.os == 'macos-13'
name: Release packages for macOS(x86_64)
uses: softprops/action-gh-release@v1
with:
files: |
jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.pkg
jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.dmg
jpackage/dist/vripper-macos-portable-${{ github.event.release.tag_name }}.x86_64.zip
jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.pkg
jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.dmg
jpackage/dist/vripper-macos-portable-${{ github.event.release.tag_name }}.zip
+5 -14
View File
@@ -1,28 +1,19 @@
# VRipper!
This is my spin for a cross-platform gallery ripper for [vipergirls.to](https://vipergirls.to)
This is my spin for a cross-platform gallery ripper for [vipergirls.to](https://vipergirls.to).
![GitHub Image](/image.png)
## Donation
To support this project, you can make a donation to its current maintainer
[!["Buy Me A Coffee"](https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png)](https://buymeacoffee.com/devclaw)
Or with Cryptocurrency
ETH: 0xDdac82B16dC5E3D742fc915ffF583D8548A301cA
BTC: bc1qcqudnkrndwyadsjwrxww42svkf8trnzx3c8vlr
## Requirements
Direct access to `vipergirls.to` domain.
## Installing VRipper
<img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/windows_logo.svg" width="100%" height="75"> Windows | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS (Intel) | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS (Apple silicon) | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/linux_logo.svg" width="100%" height="75"> Linux | <img src="https://images.vexels.com/media/users/3/166401/isolated/preview/b82aa7ac3f736dd78570dd3fa3fa9e24-java-programming-language-icon-by-vexels.png" width="100%" height="75"> Java
:---:|:---:|:---:|:---:|:---:
[Installer (EXE)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-windows-installer-6.5.0.exe) <br /> [Installer (MSI)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-windows-installer-6.5.0.msi) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-windows-portable-6.5.0.zip) | [Installer (DMG)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-macos-6.5.0.x86_64.dmg) <br /> [Installer (PKG)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-macos-6.5.0.x86_64.pkg) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-macos-portable-6.5.0.x86_64.zip) | [Installer (DMG)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-macos-6.5.0.arm64.dmg) <br /> [Installer (PKG)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-macos-6.5.0.arm64.pkg) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-macos-portable-6.5.0.arm64.zip) | [Linux (amd64) (DEB)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-linux-6.5.0_amd64.deb) <br /> [Linux (x86_64) (RPM)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-linux-6.5.0.x86_64.rpm) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-linux-portable-6.5.0.zip) | [Java GUI (noarch)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-noarch-gui-6.5.0.jar) <br /> [Java Web (noarch)](https://github.com/death-claw/vripper-project/releases/download/6.5.0/vripper-noarch-web-6.5.0.jar)
<img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/windows_logo.svg" width="100%" height="75"> Windows | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/linux_logo.svg" width="100%" height="75"> Linux | <img src="https://images.vexels.com/media/users/3/166401/isolated/preview/b82aa7ac3f736dd78570dd3fa3fa9e24-java-programming-language-icon-by-vexels.png" width="100%" height="75"> Java
:---:|:---:|:---:|:---:
[Installer (EXE)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-windows-installer-6.0.0.exe) <br /> [Installer (MSI)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-windows-installer-6.0.0.msi) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-windows-portable-6.0.0.zip) | [Installer (DMG)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-macos-6.0.0.dmg) <br /> [Installer (PKG)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-macos-6.0.0.pkg) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-macos-portable-6.0.0.zip) | [Linux (amd64) (DEB)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-linux-6.0.0_amd64.deb) <br /> [Linux (x86_64) (RPM)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-linux-6.0.0.x86_64.rpm) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-linux-portable-6.0.0.zip) | [Java GUI (noarch)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-noarch-gui-6.0.0.jar) <br /> [Java Web (noarch)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-noarch-web-6.0.0.jar)
Source code and previous versions are available on the [Releases page](https://github.com/death-claw/vripper-project/releases).
+3 -3
View File
@@ -15,7 +15,7 @@
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<revision>6.5.0</revision>
<revision>6.0.0</revision>
<kotlin.version>2.0.21</kotlin.version>
<slf4j.version>2.0.16</slf4j.version>
<logback.version>1.5.12</logback.version>
@@ -25,7 +25,7 @@
<koin.version>4.0.0</koin.version>
<h2.version>2.2.224</h2.version>
<htmlcleaner.version>2.29</htmlcleaner.version>
<failsafe.version>3.3.2</failsafe.version>
<failsafe.version>2.4.4</failsafe.version>
<caffeine.version>3.1.8</caffeine.version>
<jna-platform.version>5.14.0</jna-platform.version>
<kotlinx-serialization-json.version>1.7.3</kotlinx-serialization-json.version>
@@ -142,8 +142,8 @@
<version>${htmlcleaner.version}</version>
</dependency>
<dependency>
<groupId>dev.failsafe</groupId>
<artifactId>failsafe</artifactId>
<groupId>net.jodah</groupId>
<version>${failsafe.version}</version>
</dependency>
<dependency>
+3 -8
View File
@@ -69,7 +69,7 @@
</dependency>
<dependency>
<artifactId>failsafe</artifactId>
<groupId>dev.failsafe</groupId>
<groupId>net.jodah</groupId>
</dependency>
<dependency>
<artifactId>caffeine</artifactId>
@@ -108,11 +108,6 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-kotlin</artifactId>
</dependency>
<dependency>
<groupId>com.bucket4j</groupId>
<artifactId>bucket4j_jdk17-core</artifactId>
<version>8.14.0</version>
</dependency>
</dependencies>
<build>
@@ -141,14 +136,14 @@
<executions>
<execution>
<id>compile</id>
<phase>process-sources</phase>
<phase>compile</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
<configuration>
<jvmTarget>21</jvmTarget>
<jvmTarget>1.8</jvmTarget>
<compilerPlugins>
<plugin>kotlinx-serialization</plugin>
</compilerPlugins>
@@ -2,11 +2,11 @@ package me.vripper
import me.vripper.data.repositories.ImageRepository
import me.vripper.data.repositories.MetadataRepository
import me.vripper.data.repositories.PostRepository
import me.vripper.data.repositories.PostDownloadStateRepository
import me.vripper.data.repositories.ThreadRepository
import me.vripper.data.repositories.impl.ImageRepositoryImpl
import me.vripper.data.repositories.impl.MetadataRepositoryImpl
import me.vripper.data.repositories.impl.PostRepositoryImpl
import me.vripper.data.repositories.impl.PostDownloadStateRepositoryImpl
import me.vripper.data.repositories.impl.ThreadRepositoryImpl
import me.vripper.download.DownloadService
import me.vripper.event.EventBus
@@ -29,8 +29,8 @@ val coreModule = module {
single<ImageRepository> {
ImageRepositoryImpl()
}
single<PostRepository> {
PostRepositoryImpl()
single<PostDownloadStateRepository> {
PostDownloadStateRepositoryImpl()
}
single<MetadataRepository> {
MetadataRepositoryImpl()
@@ -69,7 +69,7 @@ val coreModule = module {
} bind IAppEndpointService::class
single<MetadataService> {
MetadataService(get(), get())
MetadataService(get(), get(), get(), get())
}
single {
AcidimgHost(get(), get(), get())
@@ -1,11 +1,12 @@
package me.vripper.data.repositories
import me.vripper.entities.PostEntity
import java.util.*
internal interface PostRepository {
internal interface PostDownloadStateRepository {
fun save(postEntities: List<PostEntity>): List<PostEntity>
fun findByPostId(postId: Long): PostEntity?
fun findById(id: Long): PostEntity?
fun findByPostId(postId: Long): Optional<PostEntity>
fun findById(id: Long): Optional<PostEntity>
fun findCompleted(): List<Long>
fun findAll(): List<PostEntity>
fun existByPostId(postId: Long): Boolean
@@ -1,6 +1,6 @@
package me.vripper.data.repositories.impl
import me.vripper.data.repositories.PostRepository
import me.vripper.data.repositories.PostDownloadStateRepository
import me.vripper.data.tables.PostTable
import me.vripper.entities.PostEntity
import me.vripper.entities.Status
@@ -8,9 +8,10 @@ import org.jetbrains.exposed.sql.*
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.transactions.TransactionManager
import java.sql.Connection
import java.util.*
internal class PostRepositoryImpl :
PostRepository {
internal class PostDownloadStateRepositoryImpl :
PostDownloadStateRepository {
private val delimiter = ";"
@@ -36,11 +37,15 @@ internal class PostRepositoryImpl :
}.map(::transform)
}
override fun findByPostId(postId: Long): PostEntity? {
override fun findByPostId(postId: Long): Optional<PostEntity> {
val result = PostTable.selectAll().where {
PostTable.postId eq postId
}.map(::transform)
return result.firstOrNull()
return if (result.isEmpty()) {
Optional.empty()
} else {
Optional.of(result.first())
}
}
override fun findCompleted(): List<Long> {
@@ -49,11 +54,16 @@ internal class PostRepositoryImpl :
}.map { it[PostTable.postId] }
}
override fun findById(id: Long): PostEntity? {
override fun findById(id: Long): Optional<PostEntity> {
val result = PostTable.selectAll().where {
PostTable.id eq id
}.map { transform(it) }
return result.firstOrNull()
return if (result.isEmpty()) {
Optional.empty()
} else {
Optional.of(result.first())
}
}
override fun findAll(): List<PostEntity> {
@@ -1,8 +1,9 @@
package me.vripper.download
import dev.failsafe.Failsafe
import dev.failsafe.RetryPolicy
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Runnable
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.launch
import me.vripper.entities.ImageEntity
import me.vripper.entities.PostEntity
import me.vripper.entities.Status
@@ -17,8 +18,10 @@ import me.vripper.services.DataTransaction
import me.vripper.services.RetryPolicyService
import me.vripper.services.SettingsService
import me.vripper.services.VGAuthService
import me.vripper.utilities.GlobalScopeCoroutine
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.downloadRunner
import net.jodah.failsafe.Failsafe
import net.jodah.failsafe.RetryPolicy
import org.jetbrains.exposed.sql.transactions.transaction
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
@@ -36,6 +39,7 @@ internal class DownloadService(
private val pending: MutableMap<Byte, MutableList<ImageDownloadRunnable>> = mutableMapOf()
private val lock = ReentrantLock()
private val condition = lock.newCondition()
private val coroutineScope = CoroutineScope(SupervisorJob())
init {
Thread.ofVirtual().name("Download Loop").unstarted(Runnable {
@@ -45,14 +49,14 @@ internal class DownloadService(
lock.withLock {
candidates.addAll(getCandidates(candidateCount()))
candidates.forEach {
if (canRun(it.imageEntity.host)) {
if (canRun(it.context.imageEntity.host)) {
accepted.add(it)
running[it.imageEntity.host]!!.add(it)
log.debug("${it.imageEntity.url} accepted to run")
running[it.context.imageEntity.host]!!.add(it)
log.debug("${it.context.imageEntity.url} accepted to run")
}
}
accepted.forEach {
pending[it.imageEntity.host]?.remove(it)
pending[it.context.imageEntity.host]?.remove(it)
scheduleForDownload(it)
}
accepted.clear()
@@ -70,10 +74,14 @@ internal class DownloadService(
fun stop(postIds: List<Long> = emptyList()) {
if (postIds.isNotEmpty()) {
stopInternal(postIds)
eventBus.publishEvent(StoppedEvent(postIds))
coroutineScope.launch {
eventBus.publishEvent(StoppedEvent(postIds))
}
} else {
stopAll()
eventBus.publishEvent(StoppedEvent(listOf(-1)))
coroutineScope.launch {
eventBus.publishEvent(StoppedEvent(listOf(-1)))
}
}
}
@@ -81,9 +89,8 @@ internal class DownloadService(
if (postEntityIds.isNotEmpty()) {
restart(postEntityIds.associateWith { dataTransaction.findByPostIdAndIsNotCompleted(it.postId) })
} else {
restart(
dataTransaction.findAllPosts()
.associateWith { dataTransaction.findByPostIdAndIsNotCompleted(it.postId) })
restart(dataTransaction.findAllPosts()
.associateWith { dataTransaction.findByPostIdAndIsNotCompleted(it.postId) })
}
}
@@ -120,6 +127,10 @@ internal class DownloadService(
)
}
toProcess.keys.forEach {
vgAuthService.leaveThanks(it)
}
toProcess.entries.forEach { (post, images) ->
images.forEach { image ->
@@ -139,18 +150,18 @@ internal class DownloadService(
}
private fun isPending(postId: Long): Boolean {
return pending.values.flatten().any { it.imageEntity.postId == postId }
return pending.values.flatten().any { it.context.imageEntity.postId == postId }
}
private fun isRunning(postId: Long): Boolean {
return running.values.flatten().any { it.imageEntity.postId == postId }
return running.values.flatten().any { it.context.imageEntity.postId == postId }
}
private fun stopAll() {
lock.withLock {
pending.values.clear()
running.values.flatten().forEach { obj: ImageDownloadRunnable -> obj.stop() }
while (running.values.flatten().count { !it.completed } > 0) {
while (running.values.flatten().count { !it.context.completed } > 0) {
Thread.sleep(100)
}
dataTransaction.findAllNonCompletedPostIds().forEach {
@@ -164,13 +175,13 @@ internal class DownloadService(
lock.withLock {
for (postId in postIds) {
pending.values.forEach { pending ->
pending.removeIf { it.imageEntity.postId == postId }
pending.removeIf { it.context.imageEntity.postId == postId }
}
running.values.flatten()
.filter { p: ImageDownloadRunnable -> p.imageEntity.postId == postId }
.filter { p: ImageDownloadRunnable -> p.context.imageEntity.postId == postId }
.forEach { obj: ImageDownloadRunnable -> obj.stop() }
while (running.values.flatten()
.count { !it.completed && it.imageEntity.postId == postId } > 0
.count { !it.context.completed && it.context.imageEntity.postId == postId } > 0
) {
Thread.sleep(100)
}
@@ -208,7 +219,7 @@ internal class DownloadService(
val list: List<ImageDownloadRunnable> =
pending[host]!!.sortedWith(Comparator.comparingInt<ImageDownloadRunnable> { it.postRank }
.thenComparingInt { it.imageEntity.index })
.thenComparingInt { it.context.imageEntity.index })
for (imageDownloadRunnable in list) {
val count = hostIntegerMap[host] ?: 0
@@ -224,42 +235,53 @@ internal class DownloadService(
}
private fun scheduleForDownload(imageDownloadRunnable: ImageDownloadRunnable) {
log.debug("Scheduling a job for ${imageDownloadRunnable.imageEntity.url}")
eventBus.publishEvent(QueueStateEvent(QueueState(runningCount(), pendingCount())))
Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicy("Failed to download ${imageDownloadRunnable.imageEntity.url}: "))
.with(downloadRunner)
.onFailure {
log.error(
"Failed to download ${imageDownloadRunnable.imageEntity.url} after ${it.attemptCount} tries",
it.exception
)
val image = imageDownloadRunnable.imageEntity
image.status = Status.ERROR
dataTransaction.updateImage(image)
log.debug("Scheduling a job for ${imageDownloadRunnable.context.imageEntity.url}")
GlobalScopeCoroutine.launch {
coroutineScope.launch {
eventBus.publishEvent(QueueStateEvent(QueueState(runningCount(), pendingCount())))
}
.onComplete {
afterJobFinish(imageDownloadRunnable)
eventBus.publishEvent(
QueueStateEvent(
QueueState(
runningCount(), pendingCount()
try {
Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicyForDownload("Failed to download ${imageDownloadRunnable.context.imageEntity.url}: "))
.onFailure {
log.error(
"Failed to download ${imageDownloadRunnable.context.imageEntity.url} after ${it.attemptCount} tries",
it.failure
)
)
)
eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataTransaction.countImagesInError())))
log.debug(
"Finished downloading ${imageDownloadRunnable.imageEntity.url}"
)
}.runAsync(imageDownloadRunnable)
val image = imageDownloadRunnable.context.imageEntity
image.status = Status.ERROR
dataTransaction.updateImage(image)
}
.onComplete {
afterJobFinish(imageDownloadRunnable)
coroutineScope.launch {
eventBus.publishEvent(
QueueStateEvent(
QueueState(
runningCount(), pendingCount()
)
)
)
}
coroutineScope.launch {
eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataTransaction.countImagesInError())))
}
log.debug(
"Finished downloading ${imageDownloadRunnable.context.imageEntity.url}"
)
}.run(imageDownloadRunnable::run)
} catch (e: Exception) {
log.error("Download Failure", e)
}
}
}
private fun afterJobFinish(imageDownloadRunnable: ImageDownloadRunnable) {
lock.withLock {
val image = imageDownloadRunnable.imageEntity
val image = imageDownloadRunnable.context.imageEntity
running[image.host]!!.remove(imageDownloadRunnable)
if (!isPending(image.postId) && !isRunning(
image.postId
) && !imageDownloadRunnable.stopped
) && !imageDownloadRunnable.context.stopped
) {
dataTransaction.finishPost(image.postId, true)
}
@@ -1,6 +1,5 @@
package me.vripper.download
import kotlinx.coroutines.*
import me.vripper.entities.ImageEntity
import me.vripper.model.Settings
import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase
@@ -9,21 +8,10 @@ import org.apache.hc.client5.http.protocol.HttpClientContext
import org.koin.core.component.KoinComponent
internal class ImageDownloadContext(val imageEntity: ImageEntity, val settings: Settings) : KoinComponent {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val jobs = mutableListOf<Job>()
val httpContext: HttpClientContext =
HttpClientContext.create().apply { cookieStore = BasicCookieStore() }
val requests = mutableListOf<HttpUriRequestBase>()
val postId = imageEntity.postIdRef
fun cancelCoroutines() {
runBlocking {
coroutineScope.cancel()
jobs.forEach { job -> job.cancelAndJoin() }
}
}
fun launchCoroutine(block: suspend CoroutineScope.() -> Unit): Job {
return coroutineScope.launch(block = block).also { job -> jobs.add(job) }
}
var stopped = false
var completed = false
}
@@ -1,6 +1,5 @@
package me.vripper.download
import dev.failsafe.function.CheckedRunnable
import me.vripper.entities.ImageEntity
import me.vripper.entities.Status
import me.vripper.exception.DownloadException
@@ -9,12 +8,12 @@ import me.vripper.host.DownloadedImage
import me.vripper.host.Host
import me.vripper.host.ImageMimeType
import me.vripper.model.Settings
import me.vripper.services.DataTransaction
import me.vripper.services.VGAuthService
import me.vripper.services.*
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.PathUtils.getExtension
import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
import me.vripper.utilities.PathUtils.sanitize
import net.jodah.failsafe.function.CheckedRunnable
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.io.IOException
@@ -26,28 +25,26 @@ import kotlin.io.path.Path
import kotlin.io.path.pathString
internal class ImageDownloadRunnable(
val imageEntity: ImageEntity, val postRank: Int, private val settings: Settings
private val imageEntity: ImageEntity, val postRank: Int, private val settings: Settings
) : KoinComponent, CheckedRunnable {
private val log by LoggerDelegate()
private val dataTransaction: DataTransaction by inject()
private val vgauthService: VGAuthService by inject()
private val hosts: List<Host> = getKoin().getAll()
var completed = false
var stopped = false
private lateinit var context: ImageDownloadContext
val context: ImageDownloadContext = ImageDownloadContext(imageEntity, settings)
@Throws(DownloadException::class)
fun download() {
try {
imageEntity.status = Status.DOWNLOADING
imageEntity.downloaded = 0
dataTransaction.updateImage(imageEntity)
synchronized(imageEntity.postId.toString().intern()) {
val post = dataTransaction.findPostById(context.postId)
val post = dataTransaction.findPostById(context.postId).orElseThrow()
if (post.status != Status.DOWNLOADING) {
post.status = Status.DOWNLOADING
dataTransaction.updatePost(post)
vgauthService.leaveThanks(post)
}
}
log.debug("Getting image url and name from ${imageEntity.url} using ${imageEntity.host}")
@@ -56,7 +53,7 @@ internal class ImageDownloadRunnable(
log.debug("Resolved name for ${imageEntity.url}: ${downloadedImage.name}")
log.debug("Downloaded image {} to {}", imageEntity.url, downloadedImage.path)
synchronized(imageEntity.postId.toString().intern()) {
val post = dataTransaction.findPostById(context.postId)
val post = dataTransaction.findPostById(context.postId).orElseThrow()
val downloadDirectory = Path(post.downloadDirectory, post.folderName).pathString
checkImageTypeAndRename(
downloadDirectory, downloadedImage, imageEntity.index
@@ -72,7 +69,7 @@ internal class ImageDownloadRunnable(
dataTransaction.updateImage(imageEntity)
}
} catch (e: Exception) {
if (stopped) {
if (context.stopped) {
return
}
imageEntity.status = Status.ERROR
@@ -121,26 +118,18 @@ internal class ImageDownloadRunnable(
}
}
@Throws(Exception::class)
override fun run() {
context = ImageDownloadContext(imageEntity, settings)
try {
if (stopped) {
if (context.stopped) {
return
}
download()
} finally {
completed = true
context.cancelCoroutines()
context.completed = true
}
}
fun stop() {
stopped = true
context.requests.forEach { it.abort() }
context.cancelCoroutines()
dataTransaction.updateImage(context.imageEntity)
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other == null || javaClass != other.javaClass) return false
@@ -151,4 +140,9 @@ internal class ImageDownloadRunnable(
override fun hashCode(): Int {
return Objects.hash(imageEntity.id)
}
fun stop() {
context.requests.forEach { it.abort() }
context.stopped = true
}
}
@@ -1,15 +1,14 @@
package me.vripper.event
import kotlinx.coroutines.channels.BufferOverflow
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.flow.asSharedFlow
object EventBus {
private val _events = MutableSharedFlow<Any>(0, Int.MAX_VALUE, BufferOverflow.DROP_OLDEST)
private val _events = MutableSharedFlow<Any>()
val events = _events.asSharedFlow()
fun publishEvent(event: Any) {
_events.tryEmit(event)
suspend fun publishEvent(event: Any) {
_events.emit(event)
}
}
@@ -3,9 +3,7 @@ package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.services.*
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
@@ -48,7 +46,7 @@ internal class AcidimgHost(
)
)
}.also { context.requests.add(it) }
log.debug(String.format("Requesting %s", httpPost.uri))
log.debug(String.format("Requesting %s", httpPost))
val doc = try {
httpService.client.execute(
httpPost, context.httpContext
@@ -1,9 +1,5 @@
package me.vripper.host
import kotlinx.coroutines.cancelAndJoin
import kotlinx.coroutines.delay
import kotlinx.coroutines.isActive
import kotlinx.coroutines.runBlocking
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.DownloadException
import me.vripper.exception.HostException
@@ -19,9 +15,10 @@ import org.apache.hc.core5.http.ClassicHttpResponse
import org.apache.hc.core5.http.Header
import org.jetbrains.exposed.sql.transactions.transaction
import org.w3c.dom.Document
import java.io.BufferedOutputStream
import java.nio.file.Files
import java.nio.file.Path
import java.time.Duration
import java.time.LocalDateTime
import kotlin.Throws
internal abstract class Host(
@@ -103,10 +100,10 @@ internal abstract class Host(
"vripper_",
".tmp"
)
return BufferedOutputStream(Files.newOutputStream(tempImage)).use { bos ->
return Files.newOutputStream(tempImage).use { fos ->
val image = context.imageEntity
synchronized(image.postId.toString().intern()) {
val post = dataTransaction.findPostById(context.postId)
val post = dataTransaction.findPostById(context.postId).orElseThrow()
val size = if (image.size < 0) {
response.entity.contentLength
} else {
@@ -127,22 +124,20 @@ internal abstract class Host(
)
val buffer = ByteArray(READ_BUFFER_SIZE)
var read: Int
val reporterJob = context.launchCoroutine {
while (isActive) {
dataTransaction.updateImage(image, false)
delay(100)
}
}
while (response.entity.content.read(buffer)
.also { read = it } != -1
var lastImageUpdateDate = LocalDateTime.now()
while (response.entity.content.read(buffer, 0, READ_BUFFER_SIZE)
.also { read = it } != -1 && !context.stopped
) {
bos.write(buffer, 0, read)
fos.write(buffer, 0, read)
image.downloaded += read
if (Duration.between(lastImageUpdateDate, LocalDateTime.now()).toMillis() > 1750) {
dataTransaction.updateImage(image)
lastImageUpdateDate = LocalDateTime.now()
} else {
dataTransaction.updateImage(image, false)
}
downloadSpeedService.reportDownloadedBytes(read.toLong())
}
runBlocking {
reporterJob.cancelAndJoin()
}
dataTransaction.updateImage(image)
Pair(tempImage, mimeType)
}
@@ -20,8 +20,6 @@ data class ViperSettings(
val password: String = "",
val thanks: Boolean = false,
val host: String = "https://vipergirls.to",
val requestLimit: Long = 2,
val fetchMetadata: Boolean = false,
)
@Serializable
@@ -1,6 +1,7 @@
package me.vripper.services
import kotlinx.coroutines.flow.*
import kotlinx.coroutines.launch
import kotlinx.coroutines.time.sample
import kotlinx.serialization.json.Json
import me.vripper.download.DownloadService
@@ -12,9 +13,9 @@ import me.vripper.tasks.AddPostTask
import me.vripper.tasks.ThreadLookupTask
import me.vripper.utilities.ApplicationProperties
import me.vripper.utilities.ApplicationProperties.VRIPPER_DIR
import me.vripper.utilities.GlobalScopeCoroutine
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.PathUtils
import me.vripper.utilities.taskRunner
import org.h2.jdbc.JdbcSQLNonTransientConnectionException
import java.sql.DriverManager
import java.time.Duration
@@ -42,10 +43,10 @@ internal class AppEndpointService(
if (postLinks.isBlank()) {
return
}
val urlList = postLinks.split(Pattern.compile("\\r?\\n")).dropLastWhile { it.isBlank() }.map { it.trim() }
val urlList = postLinks.split(Pattern.compile("\\r?\\n")).dropLastWhile { it.isEmpty() }.map { it.trim() }
.filter { it.isNotEmpty() }
for (link in urlList) {
log.debug("Scanning: $link")
log.debug("Starting to process thread: $link")
if (!link.startsWith(settingsService.settings.viperSettings.host)) {
continue
}
@@ -58,17 +59,17 @@ internal class AppEndpointService(
threadId = m.group(1).toLong()
postId = m.group(4)?.toLong()
if (postId == null) {
taskRunner.submit(
GlobalScopeCoroutine.launch {
ThreadLookupTask(
threadId, settingsService.settings
)
)
).run()
}
} else {
taskRunner.submit(
GlobalScopeCoroutine.launch {
AddPostTask(
listOf(ThreadPostId(threadId, postId))
)
)
).run()
}
}
} else {
log.error("Invalid link $link, link is missing the threadId")
@@ -80,13 +81,15 @@ internal class AppEndpointService(
override suspend fun restartAll(posIds: List<Long>) {
lock.withLock {
downloadService.restartAll(posIds.filter { dataTransaction.exists(it) }
.map { dataTransaction.findPostByPostId(it) })
downloadService.restartAll(posIds.map { dataTransaction.findPostByPostId(it) }.filter { it.isPresent }
.map { it.get() })
}
}
override suspend fun download(posts: List<ThreadPostId>) {
taskRunner.submit(AddPostTask(posts))
GlobalScopeCoroutine.launch {
AddPostTask(posts).run()
}
}
override suspend fun stopAll(postIdList: List<Long>) {
@@ -119,9 +122,6 @@ internal class AppEndpointService(
)
}
val threadLookupResult = threadCacheService[thread.threadId]
if (threadLookupResult.error.isNotBlank()) {
throw PostParseException("Error occurred for threadId = $threadId, ${threadLookupResult.error}")
}
threadLookupResult.postItemList.map { postItem ->
PostSelection(
postItem.threadId,
@@ -135,13 +135,23 @@ internal class AppEndpointService(
postItem.forum,
postItem.imageItemList.map { it.thumbLink })
}.ifEmpty {
log.error(
String.format(
"Failed to get links for threadId = %s", threadId
)
)
throw PostParseException(
"Nothing found for threadId = $threadId"
String.format(
"Failed to get links for threadId = %s", threadId
)
)
}
} catch (e: Exception) {
log.error(e.message)
throw e
throw PostParseException(
String.format(
"Failed to get links for threadId = %s, %s", threadId, e.message
)
)
}
}
}
@@ -175,21 +185,16 @@ internal class AppEndpointService(
}
override suspend fun rename(postId: Long, newName: String) {
taskRunner.submit {
GlobalScopeCoroutine.launch {
synchronized(postId.toString().intern()) {
if (dataTransaction.exists(postId)) {
dataTransaction.findPostByPostId(postId).let { post ->
if (Path(post.downloadDirectory, post.folderName).exists()) {
PathUtils.rename(
dataTransaction.findImagesByPostId(postId),
post.downloadDirectory,
post.folderName,
newName
)
}
post.folderName = PathUtils.sanitize(newName)
dataTransaction.updatePost(post)
dataTransaction.findPostByPostId(postId).ifPresent { post ->
if (Path(post.downloadDirectory, post.folderName).exists()) {
PathUtils.rename(
dataTransaction.findImagesByPostId(postId), post.downloadDirectory, post.folderName, newName
)
}
post.folderName = PathUtils.sanitize(newName)
dataTransaction.updatePost(post)
}
}
}
@@ -246,7 +251,7 @@ internal class AppEndpointService(
}
override suspend fun findPost(postId: Long): Post {
return mapper(dataTransaction.findPostByPostId(postId))
return mapper(dataTransaction.findPostByPostId(postId).orElseThrow())
}
override suspend fun findImagesByPostId(postId: Long): List<Image> {
@@ -348,7 +353,7 @@ internal class AppEndpointService(
val addedAt = it.getTimestamp("ADDED_AT")
val folderName = it.getString("FOLDER_NAME") ?: ""
val exists = dataTransaction.exists(postId)
val exists = dataTransaction.findPostByPostId(postId).isPresent
if (exists) {
continue
}
@@ -1,10 +1,12 @@
package me.vripper.services
import com.github.benmanes.caffeine.cache.Caffeine
import com.github.benmanes.caffeine.cache.LoadingCache
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.launch
import me.vripper.data.repositories.ImageRepository
import me.vripper.data.repositories.MetadataRepository
import me.vripper.data.repositories.PostRepository
import me.vripper.data.repositories.PostDownloadStateRepository
import me.vripper.data.repositories.ThreadRepository
import me.vripper.entities.*
import me.vripper.event.*
@@ -14,73 +16,70 @@ import me.vripper.utilities.PathUtils.sanitize
import me.vripper.vgapi.PostItem
import org.jetbrains.exposed.sql.transactions.transaction
import java.util.*
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicInteger
import kotlin.io.path.pathString
internal class DataTransaction(
private val settingsService: SettingsService,
private val postRepository: PostRepository,
private val postDownloadStateRepository: PostDownloadStateRepository,
private val imageRepository: ImageRepository,
private val threadRepository: ThreadRepository,
private val metadataRepository: MetadataRepository,
private val eventBus: EventBus,
) {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val nextRank = AtomicInteger(transaction { getQueuePosition() }?.plus(1) ?: 0)
private val postEntityIdCache: LoadingCache<Long, PostEntity> =
Caffeine.newBuilder().expireAfterAccess(5, TimeUnit.MINUTES).build { id ->
transaction { postRepository.findById(id) }
}
private val postPostIdCache: LoadingCache<Long, PostEntity> =
Caffeine.newBuilder().expireAfterAccess(5, TimeUnit.MINUTES).build { id ->
transaction { postRepository.findByPostId(id) }
}
private fun save(postEntities: List<PostEntity>): List<PostEntity> {
return transaction { postRepository.save(postEntities) }
return transaction { postDownloadStateRepository.save(postEntities) }
}
fun saveAndNotify(postEntity: PostEntity, images: List<ImageEntity>) {
val savedPost = transaction {
val savedPost =
postRepository.save(listOf(postEntity.copy(rank = nextRank.andIncrement))).first()
postDownloadStateRepository.save(listOf(postEntity.copy(rank = nextRank.andIncrement))).first()
save(images.map { it.copy(postIdRef = savedPost.id) })
savedPost
}
eventBus.publishEvent(PostCreateEvent(listOf(savedPost)))
coroutineScope.launch {
eventBus.publishEvent(PostCreateEvent(listOf(savedPost)))
}
}
fun updatePosts(postEntities: List<PostEntity>) {
transaction { postRepository.update(postEntities) }
postEntities.forEach { postEntity ->
postPostIdCache.put(postEntity.postId, postEntity)
postEntityIdCache.put(postEntity.id, postEntity)
transaction { postDownloadStateRepository.update(postEntities) }
coroutineScope.launch {
eventBus.publishEvent(PostUpdateEvent(postEntities))
}
eventBus.publishEvent(PostUpdateEvent(postEntities))
}
fun updatePost(postEntity: PostEntity) {
transaction { postRepository.update(postEntity) }
postPostIdCache.put(postEntity.postId, postEntity)
postEntityIdCache.put(postEntity.id, postEntity)
eventBus.publishEvent(PostUpdateEvent(listOf(postEntity)))
transaction { postDownloadStateRepository.update(postEntity) }
coroutineScope.launch {
eventBus.publishEvent(PostUpdateEvent(listOf(postEntity)))
}
}
fun save(threadEntity: ThreadEntity) {
val savedThread = transaction { threadRepository.save(threadEntity) }
eventBus.publishEvent(ThreadCreateEvent(savedThread))
coroutineScope.launch {
eventBus.publishEvent(ThreadCreateEvent(savedThread))
}
}
fun update(threadEntity: ThreadEntity) {
transaction { threadRepository.update(threadEntity) }
eventBus.publishEvent(ThreadUpdateEvent(threadEntity))
coroutineScope.launch {
eventBus.publishEvent(ThreadUpdateEvent(threadEntity))
}
}
fun updateImages(imageEntities: List<ImageEntity>) {
transaction { imageRepository.update(imageEntities) }
eventBus.publishEvent(ImageEvent(imageEntities))
coroutineScope.launch {
eventBus.publishEvent(ImageEvent(imageEntities))
}
}
fun updateImage(imageEntity: ImageEntity, persist: Boolean = true) {
@@ -89,14 +88,13 @@ internal class DataTransaction(
imageRepository.update(imageEntity)
}
}
eventBus.publishEvent(ImageEvent(listOf(imageEntity)))
coroutineScope.launch {
eventBus.publishEvent(ImageEvent(listOf(imageEntity)))
}
}
fun exists(postId: Long): Boolean {
if (postPostIdCache.getIfPresent(postId) != null) {
return true
}
return transaction { postRepository.existByPostId(postId) }
return transaction { postDownloadStateRepository.existByPostId(postId) }
}
@Synchronized
@@ -144,12 +142,14 @@ internal class DataTransaction(
}
savedPosts
}
eventBus.publishEvent(PostCreateEvent(savedPosts))
coroutineScope.launch {
eventBus.publishEvent(PostCreateEvent(savedPosts))
}
return savedPosts
}
private fun getQueuePosition(): Int? {
return postRepository.findMaxRank()
return postDownloadStateRepository.findMaxRank()
}
private fun save(imageEntities: List<ImageEntity>) {
@@ -157,7 +157,7 @@ internal class DataTransaction(
}
fun finishPost(postId: Long, automatic: Boolean = false) {
val post = findPostByPostId(postId)
val post = findPostByPostId(postId).orElseThrow()
val imagesInErrorStatus = findByPostIdAndIsError(post.postId)
if (imagesInErrorStatus.isNotEmpty()) {
post.status = Status.ERROR
@@ -188,24 +188,27 @@ internal class DataTransaction(
transaction {
metadataRepository.deleteAllByPostId(postIds)
imageRepository.deleteAllByPostId(postIds)
postRepository.deleteAll(postIds)
postDownloadStateRepository.deleteAll(postIds)
sortPostsByRank()
}
postIds.forEach { postId ->
postPostIdCache.get(postId)?.let { postEntityIdCache.invalidate(it.id) }
postPostIdCache.invalidate(postId)
coroutineScope.launch {
eventBus.publishEvent(PostDeleteEvent(postIds = postIds))
}
coroutineScope.launch {
eventBus.publishEvent(ErrorCountEvent(ErrorCount(countImagesInError())))
}
eventBus.publishEvent(PostDeleteEvent(postIds = postIds))
eventBus.publishEvent(ErrorCountEvent(ErrorCount(countImagesInError())))
}
fun removeThread(threadId: Long) {
transaction { threadRepository.deleteByThreadId(threadId) }
eventBus.publishEvent(ThreadDeleteEvent(threadId))
coroutineScope.launch {
eventBus.publishEvent(ThreadDeleteEvent(threadId))
}
}
fun clearCompleted(): List<Long> {
val completed = transaction { postRepository.findCompleted() }
val completed = transaction { postDownloadStateRepository.findCompleted() }
remove(completed)
return completed
}
@@ -228,12 +231,16 @@ internal class DataTransaction(
fun saveMetadata(metadataEntity: MetadataEntity) {
transaction { metadataRepository.save(metadataEntity) }
eventBus.publishEvent(MetadataUpdateEvent(metadataEntity))
coroutineScope.launch {
eventBus.publishEvent(MetadataUpdateEvent(metadataEntity))
}
}
fun clearQueueLinks() {
transaction { threadRepository.deleteAll() }
eventBus.publishEvent(ThreadClearEvent())
coroutineScope.launch {
eventBus.publishEvent(ThreadClearEvent())
}
}
@Synchronized
@@ -250,15 +257,15 @@ internal class DataTransaction(
}
fun setDownloadingToStopped() {
transaction { postRepository.setDownloadingToStopped() }
transaction { postDownloadStateRepository.setDownloadingToStopped() }
}
fun findAllPosts(): List<PostEntity> {
return transaction { postRepository.findAll() }
return transaction { postDownloadStateRepository.findAll() }
}
fun findPostById(id: Long): PostEntity {
return postEntityIdCache.get(id) ?: throw NoSuchElementException("Post with id = $id does not exist")
fun findPostById(id: Long): Optional<PostEntity> {
return transaction { postDownloadStateRepository.findById(id) }
}
fun findImagesByPostId(postId: Long): List<ImageEntity> {
@@ -287,8 +294,8 @@ internal class DataTransaction(
return transaction { imageRepository.countError() }
}
fun findPostByPostId(postId: Long): PostEntity {
return postPostIdCache.get(postId) ?: throw NoSuchElementException("Post with postId = $postId does not exist")
fun findPostByPostId(postId: Long): Optional<PostEntity> {
return transaction { postDownloadStateRepository.findByPostId(postId) }
}
fun findThreadByThreadId(threadId: Long): Optional<ThreadEntity> {
@@ -296,7 +303,7 @@ internal class DataTransaction(
}
fun findAllNonCompletedPostIds(): List<Long> {
return transaction { postRepository.findAllNonCompletedPostIds() }
return transaction { postDownloadStateRepository.findAllNonCompletedPostIds() }
}
fun findMetadataByPostId(postId: Long): Optional<MetadataEntity> {
@@ -30,6 +30,7 @@ internal class DownloadSpeedService(
while (isActive) {
delay(DOWNLOAD_POLL_RATE.toLong())
val newValue = bytesCount.getAndSet(0)
eventBus.publishEvent(DownloadSpeedEvent(DownloadSpeed(((newValue * 1000) / DOWNLOAD_POLL_RATE))))
}
}
@@ -14,6 +14,7 @@ import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder
import org.apache.hc.core5.pool.PoolConcurrencyPolicy
import org.apache.hc.core5.pool.PoolReusePolicy
import org.apache.hc.core5.util.TimeValue
import org.apache.hc.core5.util.Timeout
internal class HTTPService(
@@ -40,10 +41,8 @@ internal class HTTPService(
buildConnectionPool()
buildClientBuilder()
coroutineScope.launch {
while (isActive) {
pcm.closeExpired()
delay(60_000)
}
pcm.closeIdle(TimeValue.ofSeconds(60))
delay(15000)
}
coroutineScope.launch {
eventBus
@@ -65,8 +64,8 @@ internal class HTTPService(
private fun buildConnectionPool() {
pcm = PoolingHttpClientConnectionManagerBuilder.create()
.setPoolConcurrencyPolicy(PoolConcurrencyPolicy.LAX)
.setConnPoolPolicy(PoolReusePolicy.FIFO)
.setPoolConcurrencyPolicy(PoolConcurrencyPolicy.STRICT)
.setConnPoolPolicy(PoolReusePolicy.LIFO)
.setDefaultConnectionConfig(cc)
.setMaxConnTotal(Int.MAX_VALUE)
.setMaxConnPerRoute(Int.MAX_VALUE)
@@ -84,6 +83,7 @@ internal class HTTPService(
cc = ConnectionConfig.custom()
.setConnectTimeout(Timeout.ofSeconds(connectionTimeout.toLong()))
.setSocketTimeout(Timeout.ofSeconds(connectionTimeout.toLong()))
.setTimeToLive(TimeValue.ofMinutes(10))
.build()
}
@@ -1,29 +1,111 @@
package me.vripper.services
import me.vripper.tasks.FetchMetadataTask
import me.vripper.utilities.taskRunner
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.withPermit
import me.vripper.entities.MetadataEntity
import me.vripper.exception.DownloadException
import me.vripper.exception.VripperException
import me.vripper.tasks.Tasks
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.RequestLimit
import me.vripper.utilities.XpathUtils
import org.apache.hc.client5.http.classic.methods.HttpGet
import org.apache.hc.core5.http.io.entity.EntityUtils
import org.apache.hc.core5.net.URIBuilder
import org.w3c.dom.Node
import java.util.concurrent.atomic.AtomicBoolean
import java.util.stream.Collectors
internal class MetadataService(
private val dataTransaction: DataTransaction,
private val httpService: HTTPService,
private val settingsService: SettingsService,
private val dataTransaction: DataTransaction,
private val vgAuthService: VGAuthService,
) {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val dictionary: List<String> = mutableListOf("download", "link", "rapidgator", "filefactory", "filefox")
fun init() {
if (!settingsService.settings.viperSettings.fetchMetadata) {
return
}
dataTransaction.findAllPosts().filter { dataTransaction.findMetadataByPostId(it.postId).isEmpty }
.map { it.postId }.forEach(::fetchMetadata)
}
fun fetchMetadata(postId: Long) {
if (!settingsService.settings.viperSettings.fetchMetadata) {
Tasks.increment()
coroutineScope.launch {
try {
val httpGet = HttpGet(URIBuilder(settingsService.settings.viperSettings.host + "/threads/").also {
it.setParameter(
"p", postId.toString()
)
}.build())
val response = RequestLimit.semaphore.withPermit {
httpService.client.execute(httpGet, vgAuthService.context) {
if (it.code / 100 != 2) {
throw DownloadException("Unexpected response code '${it.code}' for $httpGet")
}
EntityUtils.toString(it.entity)
}
}
val document = HtmlUtils.clean(response)
val postNode: Node = XpathUtils.getAsNode(
document,
"//li[@id='post_$postId']/div[contains(@class, 'postdetails')]",
) ?: throw VripperException("Unable to find post #'$postId'")
val postedBy: String = XpathUtils.getAsNode(
postNode, "./div[contains(@class, 'userinfo')]//a[contains(@class, 'username')]//font"
)?.textContent?.trim()
?: throw VripperException("Unable to find the poster for post #'$postId'")
val node: Node = XpathUtils.getAsNode(
document, java.lang.String.format("//div[@id='post_message_%s']", postId)
) ?: throw VripperException("Unable to locate post content")
val titles = findTitleInContent(node)
val metadataEntity = MetadataEntity(postId, MetadataEntity.Data(postedBy, titles))
dataTransaction.saveMetadata(metadataEntity)
} finally {
Tasks.decrement()
}
}
}
private fun findTitleInContent(node: Node): List<String> {
val altTitle: MutableList<String> = mutableListOf()
findTitle(node, altTitle, AtomicBoolean(true))
return altTitle.stream().distinct().collect(Collectors.toList())
}
private fun findTitle(node: Node, altTitle: MutableList<String>, keepGoing: AtomicBoolean) {
if (!keepGoing.get()) {
return
}
taskRunner.submit(
FetchMetadataTask(
postId
)
)
if (node.nodeName == "a" || node.nodeName == "img") {
keepGoing.set(false)
return
}
if (node.nodeType == Node.ELEMENT_NODE) {
for (i in 0 until node.childNodes.length) {
val item = node.childNodes.item(i)
findTitle(item, altTitle, keepGoing)
if (!keepGoing.get()) {
return
}
}
} else if (node.nodeType == Node.TEXT_NODE) {
val text = node.textContent.trim()
if (text.isNotBlank() && dictionary.stream().noneMatch { e ->
text.lowercase().contains(e.lowercase())
}) {
altTitle.add(text)
}
}
}
}
@@ -1,6 +1,5 @@
package me.vripper.services
import dev.failsafe.RetryPolicy
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
@@ -9,6 +8,8 @@ import kotlinx.coroutines.launch
import me.vripper.event.EventBus
import me.vripper.event.SettingsUpdateEvent
import me.vripper.utilities.LoggerDelegate
import net.jodah.failsafe.RetryPolicy
import net.jodah.failsafe.event.ExecutionAttemptedEvent
import java.time.temporal.ChronoUnit
internal class RetryPolicyService(
@@ -28,10 +29,16 @@ internal class RetryPolicyService(
}
}
fun <T> buildRetryPolicy(message: String): RetryPolicy<T> {
return RetryPolicy.builder<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts)
.onFailedAttempt {
log.warn(message + "#${it.attemptCount} tries failed", it.lastException)
}.build()
fun <T> buildRetryPolicyForDownload(message: String): RetryPolicy<T> {
return RetryPolicy<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts).onFailedAttempt {
log.warn(message + "#${it.attemptCount} tries failed", it.lastFailure)
}
}
fun <T> buildGenericRetryPolicy(message: String): RetryPolicy<T> {
return RetryPolicy<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts)
.onFailedAttempt { e: ExecutionAttemptedEvent<T> ->
log.warn(message + "#${e.attemptCount} tries failed", e.lastFailure)
}
}
}
@@ -1,5 +1,9 @@
package me.vripper.services
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.launch
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.decodeFromStream
@@ -20,6 +24,7 @@ class SettingsService(private val eventBus: EventBus) {
private val configPath = VRIPPER_DIR.resolve("config.json")
private val customProxiesPath = VRIPPER_DIR.resolve("proxies.json")
private val proxies: MutableSet<String> = HashSet()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val json = Json {
encodeDefaults = true
@@ -35,7 +40,9 @@ class SettingsService(private val eventBus: EventBus) {
private fun init() {
loadViperProxies()
restore()
eventBus.publishEvent(SettingsUpdateEvent(settings))
coroutineScope.launch {
eventBus.publishEvent(SettingsUpdateEvent(settings))
}
}
private fun loadViperProxies() {
@@ -93,7 +100,9 @@ class SettingsService(private val eventBus: EventBus) {
}
this.settings = settings.copy(viperSettings = viperSettings)
save()
eventBus.publishEvent(SettingsUpdateEvent(this@SettingsService.settings))
coroutineScope.launch {
eventBus.publishEvent(SettingsUpdateEvent(this@SettingsService.settings))
}
}
private fun restore() {
@@ -188,11 +197,5 @@ class SettingsService(private val eventBus: EventBus) {
"Invalid clipboard monitoring polling rate settings, values must be >= 500"
)
}
if (settings.viperSettings.requestLimit < 1 || settings.viperSettings.requestLimit > 5) {
throw ValidationException(
"Invalid request rate limit, values must be in [1,5]"
)
}
}
}
@@ -2,11 +2,8 @@ package me.vripper.services
import com.github.benmanes.caffeine.cache.Caffeine
import com.github.benmanes.caffeine.cache.LoadingCache
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.*
import kotlinx.coroutines.flow.filterIsInstance
import kotlinx.coroutines.launch
import me.vripper.event.EventBus
import me.vripper.event.SettingsUpdateEvent
import me.vripper.vgapi.ThreadItem
@@ -28,7 +25,9 @@ internal class ThreadCacheService(val eventBus: EventBus) {
private val cache: LoadingCache<Long, ThreadItem> =
Caffeine.newBuilder().expireAfterWrite(20, TimeUnit.MINUTES).build { threadId ->
ThreadLookupAPIParser(threadId).parse()
runBlocking {
ThreadLookupAPIParser(threadId).parse()
}
}
@Throws(ExecutionException::class)
@@ -12,13 +12,12 @@ import me.vripper.event.VGUserLoginEvent
import me.vripper.exception.VripperException
import me.vripper.model.Settings
import me.vripper.tasks.LeaveThanksTask
import me.vripper.utilities.GlobalScopeCoroutine
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.taskRunner
import org.apache.hc.client5.http.classic.methods.HttpPost
import org.apache.hc.client5.http.cookie.BasicCookieStore
import org.apache.hc.client5.http.cookie.Cookie
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
import org.apache.hc.client5.http.impl.cookie.BasicClientCookie
import org.apache.hc.client5.http.protocol.HttpClientContext
import org.apache.hc.core5.http.io.entity.EntityUtils
import org.apache.hc.core5.http.message.BasicNameValuePair
@@ -32,25 +31,8 @@ internal class VGAuthService(
private val log by LoggerDelegate()
val context: HttpClientContext = HttpClientContext.create()
var loggedUser = ""
var authenticated = false
val clickCookies: List<Cookie>
get() {
return if (authenticated) {
val useridCookie = context.cookieStore.cookies.first { it.name.equals("vg_userid") }.let { cookie ->
BasicClientCookie(cookie.name, cookie.value).apply {
domain = "viper.click"
}
}
val passwordCookie = context.cookieStore.cookies.first { it.name.equals("vg_password") }.let { cookie ->
BasicClientCookie(cookie.name, cookie.value).apply {
domain = "viper.click"
}
}
listOf(useridCookie, passwordCookie)
} else {
emptyList()
}
}
private var authenticated = false
init {
context.cookieStore = BasicCookieStore()
@@ -68,7 +50,10 @@ internal class VGAuthService(
log.debug("Authentication option is disabled")
context.cookieStore.clear()
loggedUser = ""
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
coroutineScope.launch {
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
}
return
}
val username = settings.viperSettings.username
@@ -77,7 +62,11 @@ internal class VGAuthService(
log.error("Cannot authenticate with ViperGirls credentials, username or password is empty")
context.cookieStore.clear()
loggedUser = ""
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
coroutineScope.launch {
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
}
return
}
val postAuth = HttpPost(settings.viperSettings.host + "/login.php?do=login").also {
@@ -89,6 +78,10 @@ internal class VGAuthService(
BasicNameValuePair("vb_login_md5password", password)
)
)
it.addHeader("Referer", settings.viperSettings.host)
it.addHeader(
"Host", settings.viperSettings.host.replace("https://", "").replace("http://", "")
)
}
try {
@@ -110,7 +103,11 @@ internal class VGAuthService(
} catch (e: Exception) {
context.cookieStore.clear()
loggedUser = ""
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
coroutineScope.launch {
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
}
log.error(
"Failed to authenticate user with " + settings.viperSettings.host, e
)
@@ -118,12 +115,15 @@ internal class VGAuthService(
}
authenticated = true
loggedUser = username
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
coroutineScope.launch {
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
}
}
fun leaveThanks(postEntity: PostEntity) {
taskRunner.submit(
LeaveThanksTask(postEntity, authenticated, context)
)
GlobalScopeCoroutine.launch {
LeaveThanksTask(postEntity, authenticated, context).run()
}
}
}
@@ -1,5 +1,6 @@
package me.vripper.tasks
import kotlinx.coroutines.runBlocking
import me.vripper.download.DownloadService
import me.vripper.model.ThreadPostId
import me.vripper.services.DataTransaction
@@ -30,14 +31,14 @@ internal class AddPostTask(private val items: List<ThreadPostId>) : KoinComponen
}
val link =
"${settingsService.settings.viperSettings.host}/threads/$threadId?p=$postId&viewfull=1#post$postId"
"https://${settingsService.settings.viperSettings.host}/threads/$threadId?p=$postId&viewfull=1#post$postId"
val cachedThread = threadCacheService.getIfPresent(threadId)
val threadItem = cachedThread ?:
val threadItem = cachedThread ?: runBlocking {
PostLookupAPIParser(
threadId, postId
).parse()
}
if (threadItem == null) {
log.error("Failed to load $link")
@@ -1,110 +0,0 @@
package me.vripper.tasks
import me.vripper.entities.MetadataEntity
import me.vripper.exception.DownloadException
import me.vripper.exception.VripperException
import me.vripper.services.DataTransaction
import me.vripper.services.HTTPService
import me.vripper.services.SettingsService
import me.vripper.services.VGAuthService
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.RequestLimit
import me.vripper.utilities.XpathUtils
import org.apache.hc.client5.http.classic.methods.HttpGet
import org.apache.hc.core5.http.io.entity.EntityUtils
import org.apache.hc.core5.net.URIBuilder
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import org.w3c.dom.Node
import java.util.concurrent.atomic.AtomicBoolean
import java.util.stream.Collectors
internal class FetchMetadataTask(
private val postId: Long,
) : KoinComponent, Runnable {
private val dictionary: List<String> = mutableListOf("download", "link", "rapidgator", "filefactory", "filefox")
private val log by LoggerDelegate()
private val settingsService: SettingsService by inject()
private val vgAuthService: VGAuthService by inject()
private val httpService: HTTPService by inject()
private val dataTransaction: DataTransaction by inject()
override fun run() {
try {
if (!settingsService.settings.viperSettings.fetchMetadata) {
log.debug("Fetching metadata is disabled")
return
}
Tasks.increment()
val httpGet = HttpGet(URIBuilder(settingsService.settings.viperSettings.host + "/threads/").also {
it.setParameter(
"p", postId.toString()
)
}.build())
RequestLimit.getPermit(1)
log.debug("Requesting {}", httpGet.uri)
val response = httpService.client.execute(httpGet, vgAuthService.context) {
if (it.code / 100 != 2) {
throw DownloadException("Unexpected response code '${it.code}' for $httpGet")
}
EntityUtils.toString(it.entity)
}
val document = HtmlUtils.clean(response)
val postNode: Node = XpathUtils.getAsNode(
document,
"//li[@id='post_$postId']/div[contains(@class, 'postdetails')]",
) ?: throw VripperException("Unable to find post #'$postId'")
val postedBy: String = XpathUtils.getAsNode(
postNode, "./div[contains(@class, 'userinfo')]//a[contains(@class, 'username')]//font"
)?.textContent?.trim()
?: throw VripperException("Unable to find the poster for post #'$postId'")
val node: Node = XpathUtils.getAsNode(
document, java.lang.String.format("//div[@id='post_message_%s']", postId)
) ?: throw VripperException("Unable to locate post content")
val titles = findTitleInContent(node)
val metadataEntity = MetadataEntity(postId, MetadataEntity.Data(postedBy, titles))
dataTransaction.saveMetadata(metadataEntity)
} finally {
Tasks.decrement()
}
}
private fun findTitleInContent(node: Node): List<String> {
val altTitle: MutableList<String> = mutableListOf()
findTitle(node, altTitle, AtomicBoolean(true))
return altTitle.stream().distinct().collect(Collectors.toList())
}
private fun findTitle(node: Node, altTitle: MutableList<String>, keepGoing: AtomicBoolean) {
if (!keepGoing.get()) {
return
}
if (node.nodeName == "a" || node.nodeName == "img") {
keepGoing.set(false)
return
}
if (node.nodeType == Node.ELEMENT_NODE) {
for (i in 0 until node.childNodes.length) {
val item = node.childNodes.item(i)
findTitle(item, altTitle, keepGoing)
if (!keepGoing.get()) {
return
}
}
} else if (node.nodeType == Node.TEXT_NODE) {
val text = node.textContent.trim()
if (text.isNotBlank() && dictionary.stream().noneMatch { e ->
text.lowercase().contains(e.lowercase())
}) {
altTitle.add(text)
}
}
}
}
@@ -1,11 +1,10 @@
package me.vripper.tasks
import me.vripper.entities.PostEntity
import me.vripper.exception.VripperException
import me.vripper.services.DataTransaction
import me.vripper.services.HTTPService
import me.vripper.services.SettingsService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.RequestLimit
import org.apache.hc.client5.http.classic.methods.HttpPost
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
import org.apache.hc.client5.http.protocol.HttpClientContext
@@ -21,6 +20,7 @@ internal class LeaveThanksTask(
private val log by LoggerDelegate()
private val cm: HTTPService by inject()
private val settingsService: SettingsService by inject()
private val dataTransaction: DataTransaction by inject()
override fun run() {
try {
@@ -31,23 +31,25 @@ internal class LeaveThanksTask(
if (!settingsService.settings.viperSettings.thanks) {
return
}
val postThanks = HttpPost("${settingsService.settings.viperSettings.host}/post_thanks.php").also {
it.entity = UrlEncodedFormEntity(
listOf(
BasicNameValuePair("do", "post_thanks_add"),
BasicNameValuePair("using_ajax", "1"),
BasicNameValuePair("p", postEntity.postId.toString()),
BasicNameValuePair("securitytoken", postEntity.token)
val postThanks =
HttpPost("${settingsService.settings.viperSettings.host}/post_thanks.php").also {
it.entity = UrlEncodedFormEntity(
listOf(
BasicNameValuePair("do", "post_thanks_add"),
BasicNameValuePair("using_ajax", "1"),
BasicNameValuePair("p", postEntity.postId.toString()),
BasicNameValuePair("securitytoken", postEntity.token)
)
)
it.addHeader("Referer", settingsService.settings.viperSettings.host)
it.addHeader(
"Host",
settingsService.settings.viperSettings.host.replace("https://", "")
.replace("http://", "")
)
)
}
RequestLimit.getPermit(1)
log.debug("Requesting {}", postThanks.uri)
cm.client.execute(postThanks, context) { response ->
if (response.code / 100 != 2) {
throw VripperException("Unexpected response code '${response.code}' for $postThanks")
}
}
cm.client.execute(postThanks, context) { }
} catch (e: Exception) {
log.error("Failed to leave a thanks for $postEntity", e)
} finally {
@@ -1,5 +1,9 @@
package me.vripper.tasks
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.launch
import me.vripper.event.EventBus
import me.vripper.event.LoadingTasks
import org.koin.core.component.KoinComponent
@@ -9,11 +13,14 @@ internal object Tasks : KoinComponent {
private val eventBus: EventBus by inject()
private var current = 0
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
@Synchronized
fun increment() {
if (current == 0) {
eventBus.publishEvent(LoadingTasks(true))
coroutineScope.launch {
eventBus.publishEvent(LoadingTasks(true))
}
}
current += 1
}
@@ -22,7 +29,9 @@ internal object Tasks : KoinComponent {
fun decrement() {
current -= 1
if (current == 0) {
eventBus.publishEvent(LoadingTasks(false))
coroutineScope.launch {
eventBus.publishEvent(LoadingTasks(false))
}
}
}
}
@@ -1,13 +1,14 @@
package me.vripper.tasks
import kotlinx.coroutines.launch
import me.vripper.entities.ThreadEntity
import me.vripper.model.Settings
import me.vripper.model.ThreadPostId
import me.vripper.services.DataTransaction
import me.vripper.services.SettingsService
import me.vripper.services.ThreadCacheService
import me.vripper.utilities.GlobalScopeCoroutine
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.taskRunner
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
@@ -33,14 +34,15 @@ internal class ThreadLookupTask(private val threadId: Long, private val settings
}
if (threadLookupResult.postItemList.size <= settings.downloadSettings.autoQueueThreshold) {
taskRunner.submit(
GlobalScopeCoroutine.launch {
AddPostTask(threadLookupResult.postItemList.map {
ThreadPostId(
it.threadId, it.postId
)
})
)
}).run()
}
} else {
try {
dataTransaction.save(
ThreadEntity(
title = threadLookupResult.title,
@@ -49,6 +51,9 @@ internal class ThreadLookupTask(private val threadId: Long, private val settings
total = threadLookupResult.postItemList.size
)
)
} catch (e: Exception) {
e.printStackTrace()
}
}
}
} catch (e: Exception) {
@@ -35,7 +35,7 @@ object ApplicationProperties {
fun latestVersion(): String {
val request = HttpRequest.newBuilder()
.uri(URI.create("https://api.github.com/repos/dev-claw/vripper-project/releases/latest")).build()
.uri(URI.create("https://api.github.com/repos/death-claw/vripper-project/releases/latest")).build()
return HttpClient.newHttpClient().use {
val response = it.send(request, HttpResponse.BodyHandlers.ofString())
if (response.statusCode() / 100 != 2) {
@@ -0,0 +1,14 @@
package me.vripper.utilities
import kotlinx.coroutines.CoroutineExceptionHandler
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
private val log by LoggerDelegate()
val errorHandler = CoroutineExceptionHandler { _, exception ->
log.error("Unexpected error", exception)
}
val GlobalScopeCoroutine = CoroutineScope(SupervisorJob() + Dispatchers.IO + errorHandler)
@@ -1,42 +1,7 @@
package me.vripper.utilities
import io.github.bucket4j.Bucket
import io.github.bucket4j.BucketConfiguration
import io.github.bucket4j.TokensInheritanceStrategy
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.flow.collectLatest
import kotlinx.coroutines.flow.filterIsInstance
import kotlinx.coroutines.launch
import me.vripper.event.EventBus
import me.vripper.event.SettingsUpdateEvent
import me.vripper.services.SettingsService
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.time.Duration
import kotlinx.coroutines.sync.Semaphore
internal object RequestLimit : KoinComponent {
private val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val settingsService: SettingsService by inject()
private val eventBus: EventBus by inject()
private val bucket: Bucket = Bucket.builder().addLimit {
it.capacity(settingsService.settings.viperSettings.requestLimit)
.refillGreedy(settingsService.settings.viperSettings.requestLimit, Duration.ofSeconds(1))
}.build()
init {
coroutineScope.launch {
eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collectLatest { event ->
bucket.replaceConfiguration(BucketConfiguration.builder().addLimit {
it.capacity(event.settings.viperSettings.requestLimit)
.refillGreedy(event.settings.viperSettings.requestLimit, Duration.ofSeconds(1))
}.build(), TokensInheritanceStrategy.RESET)
}
}
}
fun getPermit(tokens: Long) {
bucket.asBlocking().consume(tokens)
}
internal object RequestLimit {
val semaphore: Semaphore = Semaphore(2)
}
@@ -1,7 +0,0 @@
package me.vripper.utilities
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
val taskRunner: ExecutorService = Executors.newFixedThreadPool(6)
val downloadRunner: ExecutorService = Executors.newVirtualThreadPerTaskExecutor()
@@ -1,26 +1,30 @@
package me.vripper.vgapi
import dev.failsafe.Failsafe
import dev.failsafe.function.CheckedSupplier
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.withPermit
import me.vripper.exception.DownloadException
import me.vripper.exception.PostParseException
import me.vripper.services.HTTPService
import me.vripper.services.RetryPolicyService
import me.vripper.services.SettingsService
import me.vripper.services.VGAuthService
import me.vripper.tasks.Tasks
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.RequestLimit
import net.jodah.failsafe.Failsafe
import net.jodah.failsafe.function.CheckedSupplier
import org.apache.hc.client5.http.classic.methods.HttpGet
import org.apache.hc.client5.http.protocol.HttpClientContext
import org.apache.hc.core5.http.io.entity.EntityUtils
import org.apache.hc.core5.net.URIBuilder
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.io.ByteArrayInputStream
import javax.xml.parsers.SAXParserFactory
import kotlin.Throws
internal class PostLookupAPIParser(private val threadId: Long, private val postId: Long) : KoinComponent {
private val log by LoggerDelegate()
private val settingsService: SettingsService by inject()
private val retryPolicyService: RetryPolicyService by inject()
private val httpService: HTTPService by inject()
private val vgAuthService: VGAuthService by inject()
@@ -29,36 +33,36 @@ internal class PostLookupAPIParser(private val threadId: Long, private val postI
fun parse(): ThreadItem? {
log.debug("Parsing post $postId")
val httpGet =
HttpGet(URIBuilder("https://viper.click/vr.php").also {
HttpGet(URIBuilder("${settingsService.settings.viperSettings.host}/vr.php").also {
it.setParameter(
"p", postId.toString()
)
}.build())
val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler()
log.debug("Requesting {}", httpGet)
Tasks.increment()
return try {
Failsafe.with(retryPolicyService.buildRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
Failsafe.with(retryPolicyService.buildGenericRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
log.error(
"Failed to process thread $threadId, post $postId", it.exception
"Failed to process thread $threadId, post $postId", it.failure
)
}.get(CheckedSupplier {
RequestLimit.getPermit(1)
log.info("Requesting {}", httpGet.uri)
httpService.client.execute(
httpGet,
HttpClientContext.create()
.apply { vgAuthService.clickCookies.forEach { cookieStore.addCookie(it) } }
) { response ->
if (response.code / 100 != 2) {
throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
}
ByteArrayInputStream(EntityUtils.toByteArray(response.entity)).use {
runBlocking {
RequestLimit.semaphore.withPermit {
httpService.client.execute(
httpGet, vgAuthService.context
) { response ->
if (response.code / 100 != 2) {
throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
}
ByteArrayInputStream(EntityUtils.toByteArray(response.entity))
}
}.use {
factory.newSAXParser()
.parse(it, threadLookupAPIResponseHandler)
threadLookupAPIResponseHandler.result
}
}
})
} catch (e: Exception) {
throw PostParseException(e)
@@ -1,62 +1,65 @@
package me.vripper.vgapi
import dev.failsafe.Failsafe
import dev.failsafe.function.CheckedSupplier
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.withPermit
import me.vripper.exception.DownloadException
import me.vripper.exception.PostParseException
import me.vripper.services.HTTPService
import me.vripper.services.RetryPolicyService
import me.vripper.services.SettingsService
import me.vripper.services.VGAuthService
import me.vripper.tasks.Tasks
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.RequestLimit
import net.jodah.failsafe.Failsafe
import net.jodah.failsafe.function.CheckedSupplier
import org.apache.hc.client5.http.classic.methods.HttpGet
import org.apache.hc.client5.http.cookie.BasicCookieStore
import org.apache.hc.client5.http.protocol.HttpClientContext
import org.apache.hc.core5.http.io.entity.EntityUtils
import org.apache.hc.core5.net.URIBuilder
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.io.ByteArrayInputStream
import javax.xml.parsers.SAXParserFactory
import kotlin.Throws
internal class ThreadLookupAPIParser(private val threadId: Long) : KoinComponent {
private val log by LoggerDelegate()
private val cm: HTTPService by inject()
private val retryPolicyService: RetryPolicyService by inject()
private val vgAuthService: VGAuthService by inject()
private val settingsService: SettingsService by inject()
@Throws(PostParseException::class)
fun parse(): ThreadItem {
log.debug("Parsing thread $threadId")
val httpGet =
HttpGet(URIBuilder("https://viper.click/vr.php").also {
HttpGet(URIBuilder(settingsService.settings.viperSettings.host + "/vr.php").also {
it.setParameter(
"t",
threadId.toString()
)
}.build())
val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler()
log.debug("Requesting {}", httpGet)
Tasks.increment()
return try {
Failsafe.with(retryPolicyService.buildRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
Failsafe.with(retryPolicyService.buildGenericRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
log.error(
"Failed to process thread $threadId",
it.exception
it.failure
)
}.get(CheckedSupplier {
RequestLimit.getPermit(1)
log.info("Requesting {}", httpGet.uri)
cm.client.execute(
httpGet, HttpClientContext.create().apply {
cookieStore = BasicCookieStore()
vgAuthService.clickCookies.forEach { cookieStore.addCookie(it) }
}
) { response ->
if (response.code / 100 != 2) {
throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
}
ByteArrayInputStream(EntityUtils.toByteArray(response.entity)).use {
runBlocking {
RequestLimit.semaphore.withPermit {
cm.client.execute(
httpGet, vgAuthService.context
) { response ->
if (response.code / 100 != 2) {
throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
}
ByteArrayInputStream(EntityUtils.toByteArray(response.entity))
}
}.use {
factory.newSAXParser().parse(
it,
threadLookupAPIResponseHandler
@@ -7,8 +7,6 @@ message ViperSettings {
string password = 3;
bool thanks = 4;
string host = 5;
int64 requestLimit = 6;
bool fetchMetadata = 7;
}
message DownloadSettings {
+2 -1
View File
@@ -19,7 +19,8 @@
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${VRIPPER_DIR}/vripper.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
<maxFileSize>10MB</maxFileSize>
<maxHistory>5</maxHistory>
<maxHistory>7</maxHistory>
<totalSizeCap>1GB</totalSizeCap>
</rollingPolicy>
</appender>
+2 -2
View File
@@ -160,14 +160,14 @@
<executions>
<execution>
<id>compile</id>
<phase>process-sources</phase>
<phase>compile</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
<configuration>
<jvmTarget>21</jvmTarget>
<jvmTarget>1.8</jvmTarget>
<compilerPlugins>
<plugin>kotlinx-serialization</plugin>
</compilerPlugins>
@@ -31,7 +31,7 @@ class VripperGuiApplication : App(
private val log by LoggerDelegate()
private var initialized = false
private val widgetsController: WidgetsController by inject()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val coroutineScope = CoroutineScope(SupervisorJob())
init {
APP_INSTANCE = this
@@ -49,8 +49,8 @@ class VripperGuiApplication : App(
with(stage) {
width = widgetsController.currentSettings.width
height = widgetsController.currentSettings.height
minWidth = 100.0
minHeight = 100.0
minWidth = 800.0
minHeight = 600.0
icons.addAll(
listOf(
Image("icons/16x16.png"),
@@ -23,7 +23,8 @@ class AboutFragment : Fragment("About") {
padding = Insets(15.0, 15.0, 15.0, 15.0)
spacing = 15.0
imageview("icons/64x64.png")
vbox(spacing = 5.0) {
vbox {
spacing = 5.0
text("VRipper") {
style {
fontWeight = FontWeight.BOLD
@@ -31,29 +32,15 @@ class AboutFragment : Fragment("About") {
}
}
text("Version ${ApplicationProperties.VERSION}")
text("Developed by dev-claw and VRipper working group")
hbox(spacing = 5.0) {
hyperlink {
imageview("icons/github-mark.png").apply {
isPreserveRatio = true
fitHeight = 32.0
}
action {
openLink("https://github.com/dev-claw/vripper-project")
}
}
hyperlink {
imageview("icons/buymeacoffee-logo.png").apply {
isPreserveRatio = true
fitHeight = 32.0
}
action {
openLink("https://buymeacoffee.com/devclaw")
}
text("Developed by death-claw and VRipper working group")
hyperlink("Home Page") {
action {
openLink("https://github.com/death-claw/vripper-project")
}
}
}
}
}
tab("System") {
form {
@@ -6,13 +6,16 @@ import javafx.geometry.Pos
import javafx.scene.control.TextArea
import javafx.scene.layout.Priority
import javafx.scene.layout.VBox
import kotlinx.coroutines.*
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancel
import kotlinx.coroutines.launch
import me.vripper.gui.controller.PostController
import tornadofx.*
class AddLinksFragment : Fragment("Add thread links") {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val coroutineScope = CoroutineScope(SupervisorJob())
private val textAreaProperty = SimpleStringProperty()
private val postController: PostController by inject()
lateinit var input: TextArea
@@ -0,0 +1,19 @@
package me.vripper.gui.components.fragments
import javafx.beans.property.SimpleBooleanProperty
import javafx.collections.FXCollections
import tornadofx.Fragment
import tornadofx.listview
import tornadofx.useCheckbox
class ColumnSelectionFragment : Fragment("Column Selection") {
val map: MutableMap<String, SimpleBooleanProperty> by param()
override val root = listview<String> {
items = FXCollections.observableArrayList(map.keys)
useCheckbox { listItem ->
map[listItem]!!
}
}
}
@@ -9,7 +9,7 @@ import tornadofx.*
class ConnectionSettingsFragment : Fragment("Connection Settings") {
private val settingsController: SettingsController by inject()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val coroutineScope = CoroutineScope(SupervisorJob())
private lateinit var connectionSettings: ConnectionSettings
val connectionSettingsModel = ConnectionSettingsModel()
@@ -13,7 +13,7 @@ class DownloadSettingsFragment : Fragment("Download Settings") {
private val settingsController: SettingsController by inject()
private val widgetsController: WidgetsController by inject()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val coroutineScope = CoroutineScope(SupervisorJob())
private lateinit var downloadSettings: DownloadSettings
val downloadSettingsModel = DownloadSettingsModel()
@@ -1,13 +1,10 @@
package me.vripper.gui.components.fragments
import atlantafx.base.theme.Styles
import javafx.beans.property.SimpleStringProperty
import javafx.geometry.Pos
import javafx.scene.control.ComboBox
import kotlinx.coroutines.*
import me.vripper.gui.controller.PostController
import org.kordamp.ikonli.feather.Feather
import org.kordamp.ikonli.javafx.FontIcon
import tornadofx.*
class RenameFragment : Fragment("Rename download post") {
@@ -38,8 +35,6 @@ class RenameFragment : Fragment("Rename download post") {
}
}
button("Rename") {
graphic = FontIcon.of(Feather.EDIT)
addClass(Styles.ACCENT)
disableWhen(comboBox.editor.textProperty().isEmpty)
action {
coroutineScope.launch {
@@ -6,7 +6,10 @@ import javafx.scene.control.Alert
import javafx.scene.control.TabPane
import javafx.scene.layout.Priority
import javafx.scene.layout.VBox
import kotlinx.coroutines.*
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancel
import kotlinx.coroutines.launch
import me.vripper.exception.ValidationException
import me.vripper.gui.controller.SettingsController
import org.kordamp.ikonli.feather.Feather
@@ -17,7 +20,7 @@ import tornadofx.*
class SettingsFragment : Fragment("Settings") {
private val settingsController: SettingsController by inject()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val coroutineScope = CoroutineScope(SupervisorJob())
private val downloadSettingsFragment: DownloadSettingsFragment = find()
private val connectionSettingsFragment: ConnectionSettingsFragment = find()
private val viperSettingsFragment: ViperSettingsFragment = find()
@@ -28,9 +31,8 @@ class SettingsFragment : Fragment("Settings") {
tabpane {
tabClosingPolicy = TabPane.TabClosingPolicy.UNAVAILABLE
VBox.setVgrow(this, Priority.ALWAYS)
minWidth = 100.0
minHeight = 100.0
prefHeight = 400.0
minWidth = 700.0
minHeight = 400.0
tab(downloadSettingsFragment.title) {
add(downloadSettingsFragment)
graphic = FontIcon.of(Feather.FOLDER)
@@ -13,7 +13,7 @@ import tornadofx.*
class SystemSettingsFragment : Fragment("System Settings") {
private val settingsController: SettingsController by inject()
private val widgetsController: WidgetsController by inject()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val coroutineScope = CoroutineScope(SupervisorJob())
private lateinit var systemSettings: SystemSettings
val systemSettingsModel = SystemSettingsModel()
@@ -2,7 +2,6 @@ package me.vripper.gui.components.fragments
import atlantafx.base.controls.ToggleSwitch
import javafx.collections.FXCollections
import javafx.scene.control.Spinner
import kotlinx.coroutines.*
import me.vripper.gui.controller.SettingsController
import me.vripper.gui.model.settings.ViperSettingsModel
@@ -12,7 +11,7 @@ import tornadofx.*
class ViperSettingsFragment : Fragment("Viper Settings") {
private val settingsController: SettingsController by inject()
private val proxies = FXCollections.observableArrayList<String>()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val coroutineScope = CoroutineScope(SupervisorJob())
private lateinit var viperGirlsSettings: ViperSettings
val viperSettingsModel = ViperSettingsModel()
@@ -26,30 +25,16 @@ class ViperSettingsFragment : Fragment("Viper Settings") {
viperSettingsModel.password = viperGirlsSettings.password
viperSettingsModel.thanks = viperGirlsSettings.thanks
viperSettingsModel.host = viperGirlsSettings.host
viperSettingsModel.requestLimit = viperGirlsSettings.requestLimit
viperSettingsModel.fetchMetadata = viperGirlsSettings.fetchMetadata
proxies.addAll(settingsController.getProxies())
}.await()
runLater {
with(root) {
form {
fieldset {
field("Viper domain") {
field("Select a proxy") {
combobox(viperSettingsModel.hostProperty, proxies)
}
field("Rate limit (requests/s)") {
add(Spinner<Int>(1, 5, viperGirlsSettings.requestLimit.toInt()).apply {
viperSettingsModel.requestLimitProperty.bind(valueProperty())
isEditable = true
atlantafx.base.util.IntegerStringConverter.createFor(this)
})
}
field("Fetch Metadata") {
checkbox {
bind(viperSettingsModel.fetchMetadataProperty)
}
}
field("Authentication") {
field("Enable ViperGirls Authentication") {
add(ToggleSwitch().apply {
isSelected = viperGirlsSettings.login
viperSettingsModel.loginProperty.bind(selectedProperty())
@@ -57,13 +42,13 @@ class ViperSettingsFragment : Fragment("Viper Settings") {
}
fieldset {
visibleWhen(viperSettingsModel.loginProperty)
field("Username") {
field("ViperGirls Username") {
textfield(viperSettingsModel.usernameProperty)
}
field("Password") {
field("ViperGirls Password") {
passwordfield(viperSettingsModel.passwordProperty)
}
field("Leave likes") {
field("Leave like") {
checkbox {
bind(viperSettingsModel.thanksProperty)
}
@@ -25,7 +25,7 @@ class ActionBarView : View() {
private val downloadActiveProperty = SimpleBooleanProperty(true)
private val postController: PostController by inject()
private val postsTableView: PostsTableView by inject()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val coroutineScope = CoroutineScope(SupervisorJob())
private val grpcEndpointService: IAppEndpointService by di("remoteAppEndpointService")
private val localEndpointService: IAppEndpointService by di("localAppEndpointService")
private val running = SimpleIntegerProperty(0)
@@ -60,10 +60,7 @@ class ActionBarView : View() {
action {
find<AddLinksFragment>().apply {
input.clear()
}.openModal()?.apply {
minWidth = 100.0
minHeight = 100.0
}
}.openModal()
}
}
separator(Orientation.VERTICAL)
@@ -114,10 +111,7 @@ class ActionBarView : View() {
contentDisplay = ContentDisplay.GRAPHIC_ONLY
tooltip("Open settings menu [Ctrl+P]")
action {
find<SettingsFragment>().openModal()?.apply {
minWidth = 100.0
minHeight = 100.0
}
find<SettingsFragment>().openModal(owner = primaryStage)
}
}
separator(Orientation.VERTICAL)
@@ -37,7 +37,7 @@ class ImagesTableView : View("Photos") {
private val widgetsController: WidgetsController by inject()
private val localAppEndpointService: IAppEndpointService by di("localAppEndpointService")
private val remoteAppEndpointService: IAppEndpointService by di("remoteAppEndpointService")
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val coroutineScope = CoroutineScope(SupervisorJob())
private val items: ObservableList<ImageModel> = FXCollections.observableArrayList()
private var preview: Preview? = null
@@ -273,9 +273,7 @@ class ImagesTableView : View("Photos") {
fun setPostId(postId: Long?) {
ActiveUICoroutines.images.forEach { it.cancel() }
ActiveUICoroutines.images.clear()
runLater {
items.clear()
}
items.clear()
if (postId == null) {
return
}
@@ -18,7 +18,7 @@ class LoadingView : View("VRipper") {
private val widgetsController: WidgetsController by inject()
private val grpcEndpointService: GrpcEndpointService by di("remoteAppEndpointService")
private val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
override val root = borderpane {}
@@ -51,12 +51,10 @@ class LoadingView : View("VRipper") {
if (!grpcEndpointService.ready()) {
val sessionView = find<SessionFragment>()
runLater {
sessionView.openModal()?.apply {
setOnCloseRequest {
sessionView.openModal().also {
it?.setOnCloseRequest {
VripperGuiApplication.APP_INSTANCE.stop()
}
minWidth = 100.0
minHeight = 100.0
}
}
}
@@ -26,7 +26,7 @@ class LogTableView : View() {
private val localAppEndpointService: IAppEndpointService by di("localAppEndpointService")
private val remoteAppEndpointService: IAppEndpointService by di("remoteAppEndpointService")
private val tableView: TableView<LogModel>
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val coroutineScope = CoroutineScope(SupervisorJob())
private val items: ObservableList<LogModel> = FXCollections.observableArrayList()
private var maxLogEvent = 0
@@ -228,8 +228,8 @@ class LogTableView : View() {
private fun openLog(item: LogModel) {
find<LogMessageFragment>(mapOf(LogMessageFragment::logModel to item)).openModal()?.apply {
minWidth = 100.0
minHeight = 100.0
minWidth = 600.0
minHeight = 400.0
}
}
}
@@ -27,7 +27,7 @@ import tornadofx.*
class MenuBarView : View() {
private val logger by LoggerDelegate()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val coroutineScope = CoroutineScope(SupervisorJob())
private val downloadActiveProperty = SimpleBooleanProperty(false)
private val postsTableView: PostsTableView by inject()
private val widgetsController: WidgetsController by inject()
@@ -67,10 +67,7 @@ class MenuBarView : View() {
action {
find<AddLinksFragment>().apply {
input.clear()
}.openModal()?.apply {
minWidth = 100.0
minHeight = 100.0
}
}.openModal()
}
}
separator()
@@ -116,20 +113,14 @@ class MenuBarView : View() {
item("Settings", KeyCodeCombination(KeyCode.P, KeyCombination.CONTROL_DOWN)).apply {
graphic = FontIcon.of(Feather.SETTINGS)
action {
find<SettingsFragment>().openModal()?.apply {
minWidth = 100.0
minHeight = 100.0
}
find<SettingsFragment>().openModal(owner = primaryStage)
}
}
separator()
item("Change session", KeyCodeCombination(KeyCode.S, KeyCombination.SHIFT_DOWN)) {
graphic = FontIcon.of(Feather.LINK_2)
action {
find<SessionFragment>().openModal()?.apply {
minWidth = 100.0
minHeight = 100.0
}
find<SessionFragment>().openModal()
}
}
separator()
@@ -197,7 +188,7 @@ class MenuBarView : View() {
owner = primaryStage,
) {
if (it == ButtonType.YES) {
openLink("https://github.com/dev-claw/vripper-project/releases/tag/$latestVersion")
openLink("https://github.com/death-claw/vripper-project/releases/tag/$latestVersion")
}
}
} else {
@@ -215,8 +206,8 @@ class MenuBarView : View() {
graphic = FontIcon.of(Feather.INFO)
action {
find<AboutFragment>().openModal()?.apply {
this.minWidth = 100.0
this.minHeight = 100.0
this.minWidth = 625.0
this.minHeight = 200.0
}
}
}
@@ -18,7 +18,7 @@ import tornadofx.*
class PostInfoView : View() {
private val logger by LoggerDelegate()
private val postController: PostController by inject()
private val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob())
private val imagesTableView: ImagesTableView by inject()
private val postModel: PostModel = PostModel(
-1, "", 0.0, "", "", 0, 0, "", "", -1, "", "", "", emptyList(), emptyList(), "", 0
@@ -37,7 +37,7 @@ import kotlin.io.path.Path
class PostsTableView : View() {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val coroutineScope = CoroutineScope(SupervisorJob())
private val postController: PostController by inject()
private val widgetsController: WidgetsController by inject()
private val clipboardService: ClipboardService by inject()
@@ -97,7 +97,7 @@ class PostsTableView : View() {
isTableMenuButtonVisible = true
primaryStage.addEventFilter(KeyEvent.KEY_PRESSED) { event ->
if (event.code == KeyCode.DELETE) {
if (isCurrentTab() && selectionModel.selectedItems.isNotEmpty() && this.isFocused) {
if (isCurrentTab() && selectionModel.selectedItems.isNotEmpty()) {
deleteSelected()
}
}
@@ -178,10 +178,7 @@ class PostsTableView : View() {
action {
find<AddLinksFragment>().apply {
input.clear()
}.openModal()?.apply {
minWidth = 100.0
minHeight = 100.0
}
}.openModal()
}
})
column("Preview", PostModel::previewListProperty) {
@@ -417,7 +414,7 @@ class PostsTableView : View() {
coroutineScope.launch {
postController.onNewPosts().collect {
runLater {
items.addAll(it)
tableView.items.addAll(it)
tableView.sort()
}
}
@@ -446,7 +443,7 @@ class PostsTableView : View() {
coroutineScope.launch {
postController.onDeletePosts().collect {
runLater {
items.items.removeIf { p -> p.postId == it }
items.removeIf { p -> p.postId == it }
tableView.sort()
}
}
@@ -472,8 +469,7 @@ class PostsTableView : View() {
RenameFragment::altTitles to post.altTitles
)
).openModal()?.apply {
minWidth = 100.0
minHeight = 100.0
minWidth = 450.0
}
}
@@ -504,7 +500,7 @@ class PostsTableView : View() {
coroutineScope.launch {
postController.delete(postIdList)
runLater {
items.items.removeIf { postIdList.contains(it.postId) }
tableView.items.removeIf { postIdList.contains(it.postId) }
}
}
}
@@ -18,7 +18,7 @@ import tornadofx.*
class StatusBarView : View("Status bar") {
private val logger by LoggerDelegate()
private val widgetsController: WidgetsController by inject()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val coroutineScope = CoroutineScope(SupervisorJob())
private val grpcEndpointService: IAppEndpointService by di("remoteAppEndpointService")
private val localEndpointService: IAppEndpointService by di("localAppEndpointService")
private val remoteText = SimpleStringProperty()
@@ -24,7 +24,7 @@ import tornadofx.*
class ThreadTableView : View() {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val coroutineScope = CoroutineScope(SupervisorJob())
private val threadController: ThreadController by inject()
private val widgetsController: WidgetsController by inject()
private val mainView: MainView by inject()
@@ -65,7 +65,7 @@ class ThreadTableView : View() {
isTableMenuButtonVisible = true
primaryStage.addEventFilter(KeyEvent.KEY_PRESSED) { event ->
if (event.code == KeyCode.DELETE) {
if (isCurrentTab() && selectionModel.selectedItems.isNotEmpty() && this.isFocused) {
if (isCurrentTab() && selectionModel.selectedItems.isNotEmpty()) {
deleteSelected()
}
}
@@ -248,8 +248,10 @@ class ThreadTableView : View() {
private fun selectPosts(threadId: Long) {
find<ThreadSelectionTableFragment>(mapOf(ThreadSelectionTableFragment::threadId to threadId)).openModal()
?.apply {
minWidth = 100.0
minHeight = 100.0
minWidth = 600.0
minHeight = 400.0
width = 800.0
height = 600.0
}
}
}
@@ -59,16 +59,14 @@ class SettingsController : Controller() {
viperSettingsModel.password,
viperSettingsModel.thanks,
viperSettingsModel.host,
viperSettingsModel.requestLimit,
viperSettingsModel.fetchMetadata,
),
systemSettings =
SystemSettings(
systemSettingsModel.tempPath,
systemSettingsModel.enable,
systemSettingsModel.pollingRate,
systemSettingsModel.logEntries
)
SystemSettings(
systemSettingsModel.tempPath,
systemSettingsModel.enable,
systemSettingsModel.pollingRate,
systemSettingsModel.logEntries
)
)
)
}
@@ -1,7 +1,6 @@
package me.vripper.gui.model.settings
import javafx.beans.property.SimpleBooleanProperty
import javafx.beans.property.SimpleLongProperty
import javafx.beans.property.SimpleStringProperty
import tornadofx.getValue
import tornadofx.setValue
@@ -21,10 +20,4 @@ class ViperSettingsModel {
val hostProperty = SimpleStringProperty()
var host: String by hostProperty
val requestLimitProperty = SimpleLongProperty()
var requestLimit: Long by requestLimitProperty
val fetchMetadataProperty = SimpleBooleanProperty()
var fetchMetadata: Boolean by fetchMetadataProperty
}
@@ -14,7 +14,7 @@ import tornadofx.runLater
class ClipboardService : Controller() {
private val logger by LoggerDelegate()
private var current: String? = null
private var coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private var coroutineScope = CoroutineScope(SupervisorJob())
private var pollJob: Job? = null
private var subscribeJob: Job? = null
@@ -176,8 +176,6 @@ class GrpcEndpointService : IAppEndpointService {
password = settings.viperSettings.password
thanks = settings.viperSettings.thanks
host = settings.viperSettings.host
requestLimit = settings.viperSettings.requestLimit
fetchMetadata = settings.viperSettings.fetchMetadata
build()
}
@@ -243,7 +241,6 @@ class GrpcEndpointService : IAppEndpointService {
password = settings.viperSettings.password,
thanks = settings.viperSettings.thanks,
host = settings.viperSettings.host,
requestLimit = settings.viperSettings.requestLimit,
),
downloadSettings = DownloadSettings(
downloadPath = settings.downloadSettings.downloadPath,
@@ -25,7 +25,7 @@ class Preview(owner: Stage, private val images: List<String>, private val cacheP
private var hBox: HBox = HBox().apply { spacing = 5.0; alignment = Pos.BOTTOM_CENTER }
private val log by LoggerDelegate()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val coroutineScope = CoroutineScope(SupervisorJob())
val previewPopup = Popup()
private val cache: LoadingCache<String, ByteArray> = Caffeine
Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.2 KiB

+1 -1
View File
@@ -106,7 +106,7 @@
<executions>
<execution>
<id>compile</id>
<phase>process-sources</phase>
<phase>compile</phase>
<goals>
<goal>compile</goal>
</goals>
@@ -196,8 +196,6 @@ class GrpcServerAppEndpointService : EndpointServiceGrpcKt.EndpointServiceCorout
password = request.viperSettings.password,
thanks = request.viperSettings.thanks,
host = request.viperSettings.host,
requestLimit = request.viperSettings.requestLimit,
fetchMetadata = request.viperSettings.fetchMetadata,
), systemSettings = SystemSettings(
tempPath = request.systemSettings.tempPath,
enableClipboardMonitoring = request.systemSettings.enableClipboardMonitoring,
@@ -230,8 +228,6 @@ class GrpcServerAppEndpointService : EndpointServiceGrpcKt.EndpointServiceCorout
password = settings.viperSettings.password
thanks = settings.viperSettings.thanks
host = settings.viperSettings.host
requestLimit = settings.viperSettings.requestLimit
fetchMetadata = settings.viperSettings.fetchMetadata
build()
}
@@ -14,7 +14,7 @@ class DataBroadcast(
private val template: SimpMessagingTemplate
) : KoinComponent {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val appEndpointService: IAppEndpointService by inject(named("localAppEndpointService"))
@OptIn(FlowPreview::class)