Compare commits

...
11 Commits
Author SHA1 Message Date
dev-claw b872d5a1a5 update links 2024-12-27 19:27:28 +01:00
dev-claw 61ce16d219 version bump 2024-12-27 19:14:46 +01:00
dev-clawandGitHub c74d711702 fixes #208 (#209) 2024-12-27 19:12:07 +01:00
death-clawandGitHub 095e69d3c9 Update README.md 2024-12-27 11:45:42 +01:00
death-clawandGitHub 60bee4df2c Update README.md 2024-12-27 11:39:59 +01:00
death-clawandGitHub 3d68691037 Update README.md 2024-12-26 22:46:03 +01:00
death-claw 4456d3d797 version bump 2024-12-25 16:24:46 +01:00
death-clawandGitHub 2bcdd81eeb fixes #206 (#207) 2024-12-25 16:23:13 +01:00
death-claw 9652c7725e version bump 2024-12-21 17:59:27 +01:00
death-clawandGitHub 01ed204d12 Update README.md 2024-12-21 17:47:27 +01:00
death-clawandGitHub b8639be241 fixes #204 (#205) 2024-12-21 17:44:28 +01:00
40 changed files with 330 additions and 370 deletions
+6 -3
View File
@@ -1,10 +1,13 @@
# VRipper!
This is my spin for a cross-platform gallery ripper for [vipergirls.to](https://vipergirls.to).
This is my spin for a cross-platform gallery ripper for [vipergirls.to](https://vipergirls.to)
![GitHub Image](/image.png)
## Donation
To support this project, you can make a donation to its current maintainer
[!["Buy Me A Coffee"](https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png)](https://buymeacoffee.com/devclaw)
## Requirements
Direct access to `vipergirls.to` domain.
@@ -13,7 +16,7 @@ Direct access to `vipergirls.to` domain.
<img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/windows_logo.svg" width="100%" height="75"> Windows | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/linux_logo.svg" width="100%" height="75"> Linux | <img src="https://images.vexels.com/media/users/3/166401/isolated/preview/b82aa7ac3f736dd78570dd3fa3fa9e24-java-programming-language-icon-by-vexels.png" width="100%" height="75"> Java
:---:|:---:|:---:|:---:
[Installer (EXE)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-windows-installer-6.0.0.exe) <br /> [Installer (MSI)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-windows-installer-6.0.0.msi) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-windows-portable-6.0.0.zip) | [Installer (DMG)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-macos-6.0.0.dmg) <br /> [Installer (PKG)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-macos-6.0.0.pkg) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-macos-portable-6.0.0.zip) | [Linux (amd64) (DEB)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-linux-6.0.0_amd64.deb) <br /> [Linux (x86_64) (RPM)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-linux-6.0.0.x86_64.rpm) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-linux-portable-6.0.0.zip) | [Java GUI (noarch)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-noarch-gui-6.0.0.jar) <br /> [Java Web (noarch)](https://github.com/death-claw/vripper-project/releases/download/6.0.0/vripper-noarch-web-6.0.0.jar)
[Installer (EXE)](https://github.com/death-claw/vripper-project/releases/download/6.3.0/vripper-windows-installer-6.3.0.exe) <br /> [Installer (MSI)](https://github.com/death-claw/vripper-project/releases/download/6.3.0/vripper-windows-installer-6.3.0.msi) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.3.0/vripper-windows-portable-6.3.0.zip) | [Installer (DMG)](https://github.com/death-claw/vripper-project/releases/download/6.3.0/vripper-macos-6.3.0.dmg) <br /> [Installer (PKG)](https://github.com/death-claw/vripper-project/releases/download/6.3.0/vripper-macos-6.3.0.pkg) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.3.0/vripper-macos-portable-6.3.0.zip) | [Linux (amd64) (DEB)](https://github.com/death-claw/vripper-project/releases/download/6.3.0/vripper-linux-6.3.0_amd64.deb) <br /> [Linux (x86_64) (RPM)](https://github.com/death-claw/vripper-project/releases/download/6.3.0/vripper-linux-6.3.0.x86_64.rpm) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.3.0/vripper-linux-portable-6.3.0.zip) | [Java GUI (noarch)](https://github.com/death-claw/vripper-project/releases/download/6.3.0/vripper-noarch-gui-6.3.0.jar) <br /> [Java Web (noarch)](https://github.com/death-claw/vripper-project/releases/download/6.3.0/vripper-noarch-web-6.3.0.jar)
Source code and previous versions are available on the [Releases page](https://github.com/death-claw/vripper-project/releases).
+3 -3
View File
@@ -15,7 +15,7 @@
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<revision>6.0.0</revision>
<revision>6.3.0</revision>
<kotlin.version>2.0.21</kotlin.version>
<slf4j.version>2.0.16</slf4j.version>
<logback.version>1.5.12</logback.version>
@@ -25,7 +25,7 @@
<koin.version>4.0.0</koin.version>
<h2.version>2.2.224</h2.version>
<htmlcleaner.version>2.29</htmlcleaner.version>
<failsafe.version>2.4.4</failsafe.version>
<failsafe.version>3.3.2</failsafe.version>
<caffeine.version>3.1.8</caffeine.version>
<jna-platform.version>5.14.0</jna-platform.version>
<kotlinx-serialization-json.version>1.7.3</kotlinx-serialization-json.version>
@@ -142,8 +142,8 @@
<version>${htmlcleaner.version}</version>
</dependency>
<dependency>
<groupId>dev.failsafe</groupId>
<artifactId>failsafe</artifactId>
<groupId>net.jodah</groupId>
<version>${failsafe.version}</version>
</dependency>
<dependency>
+1 -1
View File
@@ -69,7 +69,7 @@
</dependency>
<dependency>
<artifactId>failsafe</artifactId>
<groupId>net.jodah</groupId>
<groupId>dev.failsafe</groupId>
</dependency>
<dependency>
<artifactId>caffeine</artifactId>
@@ -2,11 +2,11 @@ package me.vripper
import me.vripper.data.repositories.ImageRepository
import me.vripper.data.repositories.MetadataRepository
import me.vripper.data.repositories.PostDownloadStateRepository
import me.vripper.data.repositories.PostRepository
import me.vripper.data.repositories.ThreadRepository
import me.vripper.data.repositories.impl.ImageRepositoryImpl
import me.vripper.data.repositories.impl.MetadataRepositoryImpl
import me.vripper.data.repositories.impl.PostDownloadStateRepositoryImpl
import me.vripper.data.repositories.impl.PostRepositoryImpl
import me.vripper.data.repositories.impl.ThreadRepositoryImpl
import me.vripper.download.DownloadService
import me.vripper.event.EventBus
@@ -29,8 +29,8 @@ val coreModule = module {
single<ImageRepository> {
ImageRepositoryImpl()
}
single<PostDownloadStateRepository> {
PostDownloadStateRepositoryImpl()
single<PostRepository> {
PostRepositoryImpl()
}
single<MetadataRepository> {
MetadataRepositoryImpl()
@@ -1,12 +1,11 @@
package me.vripper.data.repositories
import me.vripper.entities.PostEntity
import java.util.*
internal interface PostDownloadStateRepository {
internal interface PostRepository {
fun save(postEntities: List<PostEntity>): List<PostEntity>
fun findByPostId(postId: Long): Optional<PostEntity>
fun findById(id: Long): Optional<PostEntity>
fun findByPostId(postId: Long): PostEntity?
fun findById(id: Long): PostEntity?
fun findCompleted(): List<Long>
fun findAll(): List<PostEntity>
fun existByPostId(postId: Long): Boolean
@@ -1,6 +1,6 @@
package me.vripper.data.repositories.impl
import me.vripper.data.repositories.PostDownloadStateRepository
import me.vripper.data.repositories.PostRepository
import me.vripper.data.tables.PostTable
import me.vripper.entities.PostEntity
import me.vripper.entities.Status
@@ -8,10 +8,9 @@ import org.jetbrains.exposed.sql.*
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.transactions.TransactionManager
import java.sql.Connection
import java.util.*
internal class PostDownloadStateRepositoryImpl :
PostDownloadStateRepository {
internal class PostRepositoryImpl :
PostRepository {
private val delimiter = ";"
@@ -37,15 +36,11 @@ internal class PostDownloadStateRepositoryImpl :
}.map(::transform)
}
override fun findByPostId(postId: Long): Optional<PostEntity> {
override fun findByPostId(postId: Long): PostEntity? {
val result = PostTable.selectAll().where {
PostTable.postId eq postId
}.map(::transform)
return if (result.isEmpty()) {
Optional.empty()
} else {
Optional.of(result.first())
}
return result.firstOrNull()
}
override fun findCompleted(): List<Long> {
@@ -54,16 +49,11 @@ internal class PostDownloadStateRepositoryImpl :
}.map { it[PostTable.postId] }
}
override fun findById(id: Long): Optional<PostEntity> {
override fun findById(id: Long): PostEntity? {
val result = PostTable.selectAll().where {
PostTable.id eq id
}.map { transform(it) }
return if (result.isEmpty()) {
Optional.empty()
} else {
Optional.of(result.first())
}
return result.firstOrNull()
}
override fun findAll(): List<PostEntity> {
@@ -1,9 +1,8 @@
package me.vripper.download
import kotlinx.coroutines.CoroutineScope
import dev.failsafe.Failsafe
import dev.failsafe.RetryPolicy
import kotlinx.coroutines.Runnable
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.launch
import me.vripper.entities.ImageEntity
import me.vripper.entities.PostEntity
import me.vripper.entities.Status
@@ -18,10 +17,8 @@ import me.vripper.services.DataTransaction
import me.vripper.services.RetryPolicyService
import me.vripper.services.SettingsService
import me.vripper.services.VGAuthService
import me.vripper.utilities.GlobalScopeCoroutine
import me.vripper.utilities.LoggerDelegate
import net.jodah.failsafe.Failsafe
import net.jodah.failsafe.RetryPolicy
import me.vripper.utilities.executorService
import org.jetbrains.exposed.sql.transactions.transaction
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
@@ -39,7 +36,6 @@ internal class DownloadService(
private val pending: MutableMap<Byte, MutableList<ImageDownloadRunnable>> = mutableMapOf()
private val lock = ReentrantLock()
private val condition = lock.newCondition()
private val coroutineScope = CoroutineScope(SupervisorJob())
init {
Thread.ofVirtual().name("Download Loop").unstarted(Runnable {
@@ -49,14 +45,14 @@ internal class DownloadService(
lock.withLock {
candidates.addAll(getCandidates(candidateCount()))
candidates.forEach {
if (canRun(it.context.imageEntity.host)) {
if (canRun(it.imageEntity.host)) {
accepted.add(it)
running[it.context.imageEntity.host]!!.add(it)
log.debug("${it.context.imageEntity.url} accepted to run")
running[it.imageEntity.host]!!.add(it)
log.debug("${it.imageEntity.url} accepted to run")
}
}
accepted.forEach {
pending[it.context.imageEntity.host]?.remove(it)
pending[it.imageEntity.host]?.remove(it)
scheduleForDownload(it)
}
accepted.clear()
@@ -74,14 +70,10 @@ internal class DownloadService(
fun stop(postIds: List<Long> = emptyList()) {
if (postIds.isNotEmpty()) {
stopInternal(postIds)
coroutineScope.launch {
eventBus.publishEvent(StoppedEvent(postIds))
}
eventBus.publishEvent(StoppedEvent(postIds))
} else {
stopAll()
coroutineScope.launch {
eventBus.publishEvent(StoppedEvent(listOf(-1)))
}
eventBus.publishEvent(StoppedEvent(listOf(-1)))
}
}
@@ -89,8 +81,9 @@ internal class DownloadService(
if (postEntityIds.isNotEmpty()) {
restart(postEntityIds.associateWith { dataTransaction.findByPostIdAndIsNotCompleted(it.postId) })
} else {
restart(dataTransaction.findAllPosts()
.associateWith { dataTransaction.findByPostIdAndIsNotCompleted(it.postId) })
restart(
dataTransaction.findAllPosts()
.associateWith { dataTransaction.findByPostIdAndIsNotCompleted(it.postId) })
}
}
@@ -150,18 +143,18 @@ internal class DownloadService(
}
private fun isPending(postId: Long): Boolean {
return pending.values.flatten().any { it.context.imageEntity.postId == postId }
return pending.values.flatten().any { it.imageEntity.postId == postId }
}
private fun isRunning(postId: Long): Boolean {
return running.values.flatten().any { it.context.imageEntity.postId == postId }
return running.values.flatten().any { it.imageEntity.postId == postId }
}
private fun stopAll() {
lock.withLock {
pending.values.clear()
running.values.flatten().forEach { obj: ImageDownloadRunnable -> obj.stop() }
while (running.values.flatten().count { !it.context.completed } > 0) {
while (running.values.flatten().count { !it.completed } > 0) {
Thread.sleep(100)
}
dataTransaction.findAllNonCompletedPostIds().forEach {
@@ -175,13 +168,13 @@ internal class DownloadService(
lock.withLock {
for (postId in postIds) {
pending.values.forEach { pending ->
pending.removeIf { it.context.imageEntity.postId == postId }
pending.removeIf { it.imageEntity.postId == postId }
}
running.values.flatten()
.filter { p: ImageDownloadRunnable -> p.context.imageEntity.postId == postId }
.filter { p: ImageDownloadRunnable -> p.imageEntity.postId == postId }
.forEach { obj: ImageDownloadRunnable -> obj.stop() }
while (running.values.flatten()
.count { !it.context.completed && it.context.imageEntity.postId == postId } > 0
.count { !it.completed && it.imageEntity.postId == postId } > 0
) {
Thread.sleep(100)
}
@@ -219,7 +212,7 @@ internal class DownloadService(
val list: List<ImageDownloadRunnable> =
pending[host]!!.sortedWith(Comparator.comparingInt<ImageDownloadRunnable> { it.postRank }
.thenComparingInt { it.context.imageEntity.index })
.thenComparingInt { it.imageEntity.index })
for (imageDownloadRunnable in list) {
val count = hostIntegerMap[host] ?: 0
@@ -235,53 +228,42 @@ internal class DownloadService(
}
private fun scheduleForDownload(imageDownloadRunnable: ImageDownloadRunnable) {
log.debug("Scheduling a job for ${imageDownloadRunnable.context.imageEntity.url}")
GlobalScopeCoroutine.launch {
coroutineScope.launch {
eventBus.publishEvent(QueueStateEvent(QueueState(runningCount(), pendingCount())))
log.debug("Scheduling a job for ${imageDownloadRunnable.imageEntity.url}")
eventBus.publishEvent(QueueStateEvent(QueueState(runningCount(), pendingCount())))
Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicy("Failed to download ${imageDownloadRunnable.imageEntity.url}: "))
.with(executorService)
.onFailure {
log.error(
"Failed to download ${imageDownloadRunnable.imageEntity.url} after ${it.attemptCount} tries",
it.exception
)
val image = imageDownloadRunnable.imageEntity
image.status = Status.ERROR
dataTransaction.updateImage(image)
}
try {
Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicyForDownload("Failed to download ${imageDownloadRunnable.context.imageEntity.url}: "))
.onFailure {
log.error(
"Failed to download ${imageDownloadRunnable.context.imageEntity.url} after ${it.attemptCount} tries",
it.failure
.onComplete {
afterJobFinish(imageDownloadRunnable)
eventBus.publishEvent(
QueueStateEvent(
QueueState(
runningCount(), pendingCount()
)
val image = imageDownloadRunnable.context.imageEntity
image.status = Status.ERROR
dataTransaction.updateImage(image)
}
.onComplete {
afterJobFinish(imageDownloadRunnable)
coroutineScope.launch {
eventBus.publishEvent(
QueueStateEvent(
QueueState(
runningCount(), pendingCount()
)
)
)
}
coroutineScope.launch {
eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataTransaction.countImagesInError())))
}
log.debug(
"Finished downloading ${imageDownloadRunnable.context.imageEntity.url}"
)
}.run(imageDownloadRunnable::run)
} catch (e: Exception) {
log.error("Download Failure", e)
}
}
)
)
eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataTransaction.countImagesInError())))
log.debug(
"Finished downloading ${imageDownloadRunnable.imageEntity.url}"
)
}.runAsync(imageDownloadRunnable)
}
private fun afterJobFinish(imageDownloadRunnable: ImageDownloadRunnable) {
lock.withLock {
val image = imageDownloadRunnable.context.imageEntity
val image = imageDownloadRunnable.imageEntity
running[image.host]!!.remove(imageDownloadRunnable)
if (!isPending(image.postId) && !isRunning(
image.postId
) && !imageDownloadRunnable.context.stopped
) && !imageDownloadRunnable.stopped
) {
dataTransaction.finishPost(image.postId, true)
}
@@ -1,5 +1,6 @@
package me.vripper.download
import kotlinx.coroutines.*
import me.vripper.entities.ImageEntity
import me.vripper.model.Settings
import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase
@@ -8,10 +9,21 @@ import org.apache.hc.client5.http.protocol.HttpClientContext
import org.koin.core.component.KoinComponent
internal class ImageDownloadContext(val imageEntity: ImageEntity, val settings: Settings) : KoinComponent {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val jobs = mutableListOf<Job>()
val httpContext: HttpClientContext =
HttpClientContext.create().apply { cookieStore = BasicCookieStore() }
val requests = mutableListOf<HttpUriRequestBase>()
val postId = imageEntity.postIdRef
var stopped = false
var completed = false
fun cancelCoroutines() {
runBlocking {
coroutineScope.cancel()
jobs.forEach { job -> job.cancelAndJoin() }
}
}
fun launchCoroutine(block: suspend CoroutineScope.() -> Unit): Job {
return coroutineScope.launch(block = block).also { job -> jobs.add(job) }
}
}
@@ -1,5 +1,6 @@
package me.vripper.download
import dev.failsafe.function.CheckedRunnable
import me.vripper.entities.ImageEntity
import me.vripper.entities.Status
import me.vripper.exception.DownloadException
@@ -8,12 +9,11 @@ import me.vripper.host.DownloadedImage
import me.vripper.host.Host
import me.vripper.host.ImageMimeType
import me.vripper.model.Settings
import me.vripper.services.*
import me.vripper.services.DataTransaction
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.PathUtils.getExtension
import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
import me.vripper.utilities.PathUtils.sanitize
import net.jodah.failsafe.function.CheckedRunnable
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.io.IOException
@@ -21,27 +21,28 @@ import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardCopyOption
import java.util.*
import kotlin.Throws
import kotlin.io.path.Path
import kotlin.io.path.pathString
internal class ImageDownloadRunnable(
private val imageEntity: ImageEntity, val postRank: Int, private val settings: Settings
val imageEntity: ImageEntity, val postRank: Int, private val settings: Settings
) : KoinComponent, CheckedRunnable {
private val log by LoggerDelegate()
private val dataTransaction: DataTransaction by inject()
private val hosts: List<Host> = getKoin().getAll()
var completed = false
var stopped = false
val context: ImageDownloadContext = ImageDownloadContext(imageEntity, settings)
private lateinit var context: ImageDownloadContext
@Throws(DownloadException::class)
fun download() {
try {
imageEntity.status = Status.DOWNLOADING
imageEntity.downloaded = 0
dataTransaction.updateImage(imageEntity)
synchronized(imageEntity.postId.toString().intern()) {
val post = dataTransaction.findPostById(context.postId).orElseThrow()
val post = dataTransaction.findPostById(context.postId)
if (post.status != Status.DOWNLOADING) {
post.status = Status.DOWNLOADING
dataTransaction.updatePost(post)
@@ -53,7 +54,7 @@ internal class ImageDownloadRunnable(
log.debug("Resolved name for ${imageEntity.url}: ${downloadedImage.name}")
log.debug("Downloaded image {} to {}", imageEntity.url, downloadedImage.path)
synchronized(imageEntity.postId.toString().intern()) {
val post = dataTransaction.findPostById(context.postId).orElseThrow()
val post = dataTransaction.findPostById(context.postId)
val downloadDirectory = Path(post.downloadDirectory, post.folderName).pathString
checkImageTypeAndRename(
downloadDirectory, downloadedImage, imageEntity.index
@@ -69,7 +70,7 @@ internal class ImageDownloadRunnable(
dataTransaction.updateImage(imageEntity)
}
} catch (e: Exception) {
if (context.stopped) {
if (stopped) {
return
}
imageEntity.status = Status.ERROR
@@ -118,18 +119,26 @@ internal class ImageDownloadRunnable(
}
}
@Throws(Exception::class)
override fun run() {
context = ImageDownloadContext(imageEntity, settings)
try {
if (context.stopped) {
if (stopped) {
return
}
download()
} finally {
context.completed = true
completed = true
context.cancelCoroutines()
}
}
fun stop() {
stopped = true
context.requests.forEach { it.abort() }
context.cancelCoroutines()
dataTransaction.updateImage(context.imageEntity)
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other == null || javaClass != other.javaClass) return false
@@ -140,9 +149,4 @@ internal class ImageDownloadRunnable(
override fun hashCode(): Int {
return Objects.hash(imageEntity.id)
}
fun stop() {
context.requests.forEach { it.abort() }
context.stopped = true
}
}
@@ -1,14 +1,15 @@
package me.vripper.event
import kotlinx.coroutines.channels.BufferOverflow
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.flow.asSharedFlow
object EventBus {
private val _events = MutableSharedFlow<Any>()
private val _events = MutableSharedFlow<Any>(0, Int.MAX_VALUE, BufferOverflow.DROP_OLDEST)
val events = _events.asSharedFlow()
suspend fun publishEvent(event: Any) {
_events.emit(event)
fun publishEvent(event: Any) {
_events.tryEmit(event)
}
}
@@ -1,5 +1,9 @@
package me.vripper.host
import kotlinx.coroutines.cancelAndJoin
import kotlinx.coroutines.delay
import kotlinx.coroutines.isActive
import kotlinx.coroutines.runBlocking
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.DownloadException
import me.vripper.exception.HostException
@@ -15,10 +19,9 @@ import org.apache.hc.core5.http.ClassicHttpResponse
import org.apache.hc.core5.http.Header
import org.jetbrains.exposed.sql.transactions.transaction
import org.w3c.dom.Document
import java.io.BufferedOutputStream
import java.nio.file.Files
import java.nio.file.Path
import java.time.Duration
import java.time.LocalDateTime
import kotlin.Throws
internal abstract class Host(
@@ -100,10 +103,10 @@ internal abstract class Host(
"vripper_",
".tmp"
)
return Files.newOutputStream(tempImage).use { fos ->
return BufferedOutputStream(Files.newOutputStream(tempImage)).use { bos ->
val image = context.imageEntity
synchronized(image.postId.toString().intern()) {
val post = dataTransaction.findPostById(context.postId).orElseThrow()
val post = dataTransaction.findPostById(context.postId)
val size = if (image.size < 0) {
response.entity.contentLength
} else {
@@ -124,20 +127,22 @@ internal abstract class Host(
)
val buffer = ByteArray(READ_BUFFER_SIZE)
var read: Int
var lastImageUpdateDate = LocalDateTime.now()
while (response.entity.content.read(buffer, 0, READ_BUFFER_SIZE)
.also { read = it } != -1 && !context.stopped
) {
fos.write(buffer, 0, read)
image.downloaded += read
if (Duration.between(lastImageUpdateDate, LocalDateTime.now()).toMillis() > 1750) {
dataTransaction.updateImage(image)
lastImageUpdateDate = LocalDateTime.now()
} else {
val reporterJob = context.launchCoroutine {
while (isActive) {
dataTransaction.updateImage(image, false)
delay(100)
}
}
while (response.entity.content.read(buffer)
.also { read = it } != -1
) {
bos.write(buffer, 0, read)
image.downloaded += read
downloadSpeedService.reportDownloadedBytes(read.toLong())
}
runBlocking {
reporterJob.cancelAndJoin()
}
dataTransaction.updateImage(image)
Pair(tempImage, mimeType)
}
@@ -1,7 +1,6 @@
package me.vripper.services
import kotlinx.coroutines.flow.*
import kotlinx.coroutines.launch
import kotlinx.coroutines.time.sample
import kotlinx.serialization.json.Json
import me.vripper.download.DownloadService
@@ -13,9 +12,9 @@ import me.vripper.tasks.AddPostTask
import me.vripper.tasks.ThreadLookupTask
import me.vripper.utilities.ApplicationProperties
import me.vripper.utilities.ApplicationProperties.VRIPPER_DIR
import me.vripper.utilities.GlobalScopeCoroutine
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.PathUtils
import me.vripper.utilities.executorService
import org.h2.jdbc.JdbcSQLNonTransientConnectionException
import java.sql.DriverManager
import java.time.Duration
@@ -59,17 +58,17 @@ internal class AppEndpointService(
threadId = m.group(1).toLong()
postId = m.group(4)?.toLong()
if (postId == null) {
GlobalScopeCoroutine.launch {
executorService.submit(
ThreadLookupTask(
threadId, settingsService.settings
).run()
}
)
)
} else {
GlobalScopeCoroutine.launch {
executorService.submit(
AddPostTask(
listOf(ThreadPostId(threadId, postId))
).run()
}
)
)
}
} else {
log.error("Invalid link $link, link is missing the threadId")
@@ -81,15 +80,13 @@ internal class AppEndpointService(
override suspend fun restartAll(posIds: List<Long>) {
lock.withLock {
downloadService.restartAll(posIds.map { dataTransaction.findPostByPostId(it) }.filter { it.isPresent }
.map { it.get() })
downloadService.restartAll(posIds.filter { dataTransaction.exists(it) }
.map { dataTransaction.findPostByPostId(it) })
}
}
override suspend fun download(posts: List<ThreadPostId>) {
GlobalScopeCoroutine.launch {
AddPostTask(posts).run()
}
executorService.submit(AddPostTask(posts))
}
override suspend fun stopAll(postIdList: List<Long>) {
@@ -185,16 +182,21 @@ internal class AppEndpointService(
}
override suspend fun rename(postId: Long, newName: String) {
GlobalScopeCoroutine.launch {
executorService.submit {
synchronized(postId.toString().intern()) {
dataTransaction.findPostByPostId(postId).ifPresent { post ->
if (Path(post.downloadDirectory, post.folderName).exists()) {
PathUtils.rename(
dataTransaction.findImagesByPostId(postId), post.downloadDirectory, post.folderName, newName
)
if (dataTransaction.exists(postId)) {
dataTransaction.findPostByPostId(postId).let { post ->
if (Path(post.downloadDirectory, post.folderName).exists()) {
PathUtils.rename(
dataTransaction.findImagesByPostId(postId),
post.downloadDirectory,
post.folderName,
newName
)
}
post.folderName = PathUtils.sanitize(newName)
dataTransaction.updatePost(post)
}
post.folderName = PathUtils.sanitize(newName)
dataTransaction.updatePost(post)
}
}
}
@@ -251,7 +253,7 @@ internal class AppEndpointService(
}
override suspend fun findPost(postId: Long): Post {
return mapper(dataTransaction.findPostByPostId(postId).orElseThrow())
return mapper(dataTransaction.findPostByPostId(postId))
}
override suspend fun findImagesByPostId(postId: Long): List<Image> {
@@ -353,7 +355,7 @@ internal class AppEndpointService(
val addedAt = it.getTimestamp("ADDED_AT")
val folderName = it.getString("FOLDER_NAME") ?: ""
val exists = dataTransaction.findPostByPostId(postId).isPresent
val exists = dataTransaction.exists(postId)
if (exists) {
continue
}
@@ -1,12 +1,10 @@
package me.vripper.services
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.launch
import com.github.benmanes.caffeine.cache.Caffeine
import com.github.benmanes.caffeine.cache.LoadingCache
import me.vripper.data.repositories.ImageRepository
import me.vripper.data.repositories.MetadataRepository
import me.vripper.data.repositories.PostDownloadStateRepository
import me.vripper.data.repositories.PostRepository
import me.vripper.data.repositories.ThreadRepository
import me.vripper.entities.*
import me.vripper.event.*
@@ -16,70 +14,73 @@ import me.vripper.utilities.PathUtils.sanitize
import me.vripper.vgapi.PostItem
import org.jetbrains.exposed.sql.transactions.transaction
import java.util.*
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicInteger
import kotlin.io.path.pathString
internal class DataTransaction(
private val settingsService: SettingsService,
private val postDownloadStateRepository: PostDownloadStateRepository,
private val postRepository: PostRepository,
private val imageRepository: ImageRepository,
private val threadRepository: ThreadRepository,
private val metadataRepository: MetadataRepository,
private val eventBus: EventBus,
) {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val nextRank = AtomicInteger(transaction { getQueuePosition() }?.plus(1) ?: 0)
private val postEntityIdCache: LoadingCache<Long, PostEntity> =
Caffeine.newBuilder().expireAfterAccess(5, TimeUnit.MINUTES).build { id ->
transaction { postRepository.findById(id) }
}
private val postPostIdCache: LoadingCache<Long, PostEntity> =
Caffeine.newBuilder().expireAfterAccess(5, TimeUnit.MINUTES).build { id ->
transaction { postRepository.findByPostId(id) }
}
private fun save(postEntities: List<PostEntity>): List<PostEntity> {
return transaction { postDownloadStateRepository.save(postEntities) }
return transaction { postRepository.save(postEntities) }
}
fun saveAndNotify(postEntity: PostEntity, images: List<ImageEntity>) {
val savedPost = transaction {
val savedPost =
postDownloadStateRepository.save(listOf(postEntity.copy(rank = nextRank.andIncrement))).first()
postRepository.save(listOf(postEntity.copy(rank = nextRank.andIncrement))).first()
save(images.map { it.copy(postIdRef = savedPost.id) })
savedPost
}
coroutineScope.launch {
eventBus.publishEvent(PostCreateEvent(listOf(savedPost)))
}
eventBus.publishEvent(PostCreateEvent(listOf(savedPost)))
}
fun updatePosts(postEntities: List<PostEntity>) {
transaction { postDownloadStateRepository.update(postEntities) }
coroutineScope.launch {
eventBus.publishEvent(PostUpdateEvent(postEntities))
transaction { postRepository.update(postEntities) }
postEntities.forEach { postEntity ->
postPostIdCache.put(postEntity.postId, postEntity)
postEntityIdCache.put(postEntity.id, postEntity)
}
eventBus.publishEvent(PostUpdateEvent(postEntities))
}
fun updatePost(postEntity: PostEntity) {
transaction { postDownloadStateRepository.update(postEntity) }
coroutineScope.launch {
eventBus.publishEvent(PostUpdateEvent(listOf(postEntity)))
}
transaction { postRepository.update(postEntity) }
postPostIdCache.put(postEntity.postId, postEntity)
postEntityIdCache.put(postEntity.id, postEntity)
eventBus.publishEvent(PostUpdateEvent(listOf(postEntity)))
}
fun save(threadEntity: ThreadEntity) {
val savedThread = transaction { threadRepository.save(threadEntity) }
coroutineScope.launch {
eventBus.publishEvent(ThreadCreateEvent(savedThread))
}
eventBus.publishEvent(ThreadCreateEvent(savedThread))
}
fun update(threadEntity: ThreadEntity) {
transaction { threadRepository.update(threadEntity) }
coroutineScope.launch {
eventBus.publishEvent(ThreadUpdateEvent(threadEntity))
}
eventBus.publishEvent(ThreadUpdateEvent(threadEntity))
}
fun updateImages(imageEntities: List<ImageEntity>) {
transaction { imageRepository.update(imageEntities) }
coroutineScope.launch {
eventBus.publishEvent(ImageEvent(imageEntities))
}
eventBus.publishEvent(ImageEvent(imageEntities))
}
fun updateImage(imageEntity: ImageEntity, persist: Boolean = true) {
@@ -88,13 +89,14 @@ internal class DataTransaction(
imageRepository.update(imageEntity)
}
}
coroutineScope.launch {
eventBus.publishEvent(ImageEvent(listOf(imageEntity)))
}
eventBus.publishEvent(ImageEvent(listOf(imageEntity)))
}
fun exists(postId: Long): Boolean {
return transaction { postDownloadStateRepository.existByPostId(postId) }
if (postPostIdCache.getIfPresent(postId) != null) {
return true
}
return transaction { postRepository.existByPostId(postId) }
}
@Synchronized
@@ -142,14 +144,12 @@ internal class DataTransaction(
}
savedPosts
}
coroutineScope.launch {
eventBus.publishEvent(PostCreateEvent(savedPosts))
}
eventBus.publishEvent(PostCreateEvent(savedPosts))
return savedPosts
}
private fun getQueuePosition(): Int? {
return postDownloadStateRepository.findMaxRank()
return postRepository.findMaxRank()
}
private fun save(imageEntities: List<ImageEntity>) {
@@ -157,7 +157,7 @@ internal class DataTransaction(
}
fun finishPost(postId: Long, automatic: Boolean = false) {
val post = findPostByPostId(postId).orElseThrow()
val post = findPostByPostId(postId)
val imagesInErrorStatus = findByPostIdAndIsError(post.postId)
if (imagesInErrorStatus.isNotEmpty()) {
post.status = Status.ERROR
@@ -188,27 +188,24 @@ internal class DataTransaction(
transaction {
metadataRepository.deleteAllByPostId(postIds)
imageRepository.deleteAllByPostId(postIds)
postDownloadStateRepository.deleteAll(postIds)
postRepository.deleteAll(postIds)
sortPostsByRank()
}
coroutineScope.launch {
eventBus.publishEvent(PostDeleteEvent(postIds = postIds))
}
coroutineScope.launch {
eventBus.publishEvent(ErrorCountEvent(ErrorCount(countImagesInError())))
postIds.forEach { postId ->
postPostIdCache.get(postId)?.let { postEntityIdCache.invalidate(it.id) }
postPostIdCache.invalidate(postId)
}
eventBus.publishEvent(PostDeleteEvent(postIds = postIds))
eventBus.publishEvent(ErrorCountEvent(ErrorCount(countImagesInError())))
}
fun removeThread(threadId: Long) {
transaction { threadRepository.deleteByThreadId(threadId) }
coroutineScope.launch {
eventBus.publishEvent(ThreadDeleteEvent(threadId))
}
eventBus.publishEvent(ThreadDeleteEvent(threadId))
}
fun clearCompleted(): List<Long> {
val completed = transaction { postDownloadStateRepository.findCompleted() }
val completed = transaction { postRepository.findCompleted() }
remove(completed)
return completed
}
@@ -231,16 +228,12 @@ internal class DataTransaction(
fun saveMetadata(metadataEntity: MetadataEntity) {
transaction { metadataRepository.save(metadataEntity) }
coroutineScope.launch {
eventBus.publishEvent(MetadataUpdateEvent(metadataEntity))
}
eventBus.publishEvent(MetadataUpdateEvent(metadataEntity))
}
fun clearQueueLinks() {
transaction { threadRepository.deleteAll() }
coroutineScope.launch {
eventBus.publishEvent(ThreadClearEvent())
}
eventBus.publishEvent(ThreadClearEvent())
}
@Synchronized
@@ -257,15 +250,15 @@ internal class DataTransaction(
}
fun setDownloadingToStopped() {
transaction { postDownloadStateRepository.setDownloadingToStopped() }
transaction { postRepository.setDownloadingToStopped() }
}
fun findAllPosts(): List<PostEntity> {
return transaction { postDownloadStateRepository.findAll() }
return transaction { postRepository.findAll() }
}
fun findPostById(id: Long): Optional<PostEntity> {
return transaction { postDownloadStateRepository.findById(id) }
fun findPostById(id: Long): PostEntity {
return postEntityIdCache.get(id) ?: throw NoSuchElementException("Post with id = $id does not exist")
}
fun findImagesByPostId(postId: Long): List<ImageEntity> {
@@ -294,8 +287,8 @@ internal class DataTransaction(
return transaction { imageRepository.countError() }
}
fun findPostByPostId(postId: Long): Optional<PostEntity> {
return transaction { postDownloadStateRepository.findByPostId(postId) }
fun findPostByPostId(postId: Long): PostEntity {
return postPostIdCache.get(postId) ?: throw NoSuchElementException("Post with postId = $postId does not exist")
}
fun findThreadByThreadId(threadId: Long): Optional<ThreadEntity> {
@@ -303,7 +296,7 @@ internal class DataTransaction(
}
fun findAllNonCompletedPostIds(): List<Long> {
return transaction { postDownloadStateRepository.findAllNonCompletedPostIds() }
return transaction { postRepository.findAllNonCompletedPostIds() }
}
fun findMetadataByPostId(postId: Long): Optional<MetadataEntity> {
@@ -30,7 +30,6 @@ internal class DownloadSpeedService(
while (isActive) {
delay(DOWNLOAD_POLL_RATE.toLong())
val newValue = bytesCount.getAndSet(0)
eventBus.publishEvent(DownloadSpeedEvent(DownloadSpeed(((newValue * 1000) / DOWNLOAD_POLL_RATE))))
}
}
@@ -14,7 +14,6 @@ import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder
import org.apache.hc.core5.pool.PoolConcurrencyPolicy
import org.apache.hc.core5.pool.PoolReusePolicy
import org.apache.hc.core5.util.TimeValue
import org.apache.hc.core5.util.Timeout
internal class HTTPService(
@@ -41,8 +40,10 @@ internal class HTTPService(
buildConnectionPool()
buildClientBuilder()
coroutineScope.launch {
pcm.closeIdle(TimeValue.ofSeconds(60))
delay(15000)
while (isActive) {
pcm.closeExpired()
delay(60_000)
}
}
coroutineScope.launch {
eventBus
@@ -64,8 +65,8 @@ internal class HTTPService(
private fun buildConnectionPool() {
pcm = PoolingHttpClientConnectionManagerBuilder.create()
.setPoolConcurrencyPolicy(PoolConcurrencyPolicy.STRICT)
.setConnPoolPolicy(PoolReusePolicy.LIFO)
.setPoolConcurrencyPolicy(PoolConcurrencyPolicy.LAX)
.setConnPoolPolicy(PoolReusePolicy.FIFO)
.setDefaultConnectionConfig(cc)
.setMaxConnTotal(Int.MAX_VALUE)
.setMaxConnPerRoute(Int.MAX_VALUE)
@@ -83,7 +84,6 @@ internal class HTTPService(
cc = ConnectionConfig.custom()
.setConnectTimeout(Timeout.ofSeconds(connectionTimeout.toLong()))
.setSocketTimeout(Timeout.ofSeconds(connectionTimeout.toLong()))
.setTimeToLive(TimeValue.ofMinutes(10))
.build()
}
@@ -1,5 +1,6 @@
package me.vripper.services
import dev.failsafe.RetryPolicy
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
@@ -8,8 +9,6 @@ import kotlinx.coroutines.launch
import me.vripper.event.EventBus
import me.vripper.event.SettingsUpdateEvent
import me.vripper.utilities.LoggerDelegate
import net.jodah.failsafe.RetryPolicy
import net.jodah.failsafe.event.ExecutionAttemptedEvent
import java.time.temporal.ChronoUnit
internal class RetryPolicyService(
@@ -29,16 +28,10 @@ internal class RetryPolicyService(
}
}
fun <T> buildRetryPolicyForDownload(message: String): RetryPolicy<T> {
return RetryPolicy<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts).onFailedAttempt {
log.warn(message + "#${it.attemptCount} tries failed", it.lastFailure)
}
}
fun <T> buildGenericRetryPolicy(message: String): RetryPolicy<T> {
return RetryPolicy<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts)
.onFailedAttempt { e: ExecutionAttemptedEvent<T> ->
log.warn(message + "#${e.attemptCount} tries failed", e.lastFailure)
}
fun <T> buildRetryPolicy(message: String): RetryPolicy<T> {
return RetryPolicy.builder<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts)
.onFailedAttempt {
log.warn(message + "#${it.attemptCount} tries failed", it.lastException)
}.build()
}
}
@@ -1,9 +1,5 @@
package me.vripper.services
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.launch
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.decodeFromStream
@@ -24,7 +20,6 @@ class SettingsService(private val eventBus: EventBus) {
private val configPath = VRIPPER_DIR.resolve("config.json")
private val customProxiesPath = VRIPPER_DIR.resolve("proxies.json")
private val proxies: MutableSet<String> = HashSet()
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val json = Json {
encodeDefaults = true
@@ -40,9 +35,7 @@ class SettingsService(private val eventBus: EventBus) {
private fun init() {
loadViperProxies()
restore()
coroutineScope.launch {
eventBus.publishEvent(SettingsUpdateEvent(settings))
}
eventBus.publishEvent(SettingsUpdateEvent(settings))
}
private fun loadViperProxies() {
@@ -100,9 +93,7 @@ class SettingsService(private val eventBus: EventBus) {
}
this.settings = settings.copy(viperSettings = viperSettings)
save()
coroutineScope.launch {
eventBus.publishEvent(SettingsUpdateEvent(this@SettingsService.settings))
}
eventBus.publishEvent(SettingsUpdateEvent(this@SettingsService.settings))
}
private fun restore() {
@@ -2,8 +2,11 @@ package me.vripper.services
import com.github.benmanes.caffeine.cache.Caffeine
import com.github.benmanes.caffeine.cache.LoadingCache
import kotlinx.coroutines.*
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.flow.filterIsInstance
import kotlinx.coroutines.launch
import me.vripper.event.EventBus
import me.vripper.event.SettingsUpdateEvent
import me.vripper.vgapi.ThreadItem
@@ -25,9 +28,7 @@ internal class ThreadCacheService(val eventBus: EventBus) {
private val cache: LoadingCache<Long, ThreadItem> =
Caffeine.newBuilder().expireAfterWrite(20, TimeUnit.MINUTES).build { threadId ->
runBlocking {
ThreadLookupAPIParser(threadId).parse()
}
ThreadLookupAPIParser(threadId).parse()
}
@Throws(ExecutionException::class)
@@ -12,8 +12,8 @@ import me.vripper.event.VGUserLoginEvent
import me.vripper.exception.VripperException
import me.vripper.model.Settings
import me.vripper.tasks.LeaveThanksTask
import me.vripper.utilities.GlobalScopeCoroutine
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.executorService
import org.apache.hc.client5.http.classic.methods.HttpPost
import org.apache.hc.client5.http.cookie.BasicCookieStore
import org.apache.hc.client5.http.cookie.Cookie
@@ -50,10 +50,7 @@ internal class VGAuthService(
log.debug("Authentication option is disabled")
context.cookieStore.clear()
loggedUser = ""
coroutineScope.launch {
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
}
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
return
}
val username = settings.viperSettings.username
@@ -62,11 +59,7 @@ internal class VGAuthService(
log.error("Cannot authenticate with ViperGirls credentials, username or password is empty")
context.cookieStore.clear()
loggedUser = ""
coroutineScope.launch {
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
}
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
return
}
val postAuth = HttpPost(settings.viperSettings.host + "/login.php?do=login").also {
@@ -103,11 +96,7 @@ internal class VGAuthService(
} catch (e: Exception) {
context.cookieStore.clear()
loggedUser = ""
coroutineScope.launch {
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
}
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
log.error(
"Failed to authenticate user with " + settings.viperSettings.host, e
)
@@ -115,15 +104,12 @@ internal class VGAuthService(
}
authenticated = true
loggedUser = username
coroutineScope.launch {
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
}
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
}
fun leaveThanks(postEntity: PostEntity) {
GlobalScopeCoroutine.launch {
LeaveThanksTask(postEntity, authenticated, context).run()
}
executorService.submit(
LeaveThanksTask(postEntity, authenticated, context)
)
}
}
@@ -1,9 +1,5 @@
package me.vripper.tasks
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.launch
import me.vripper.event.EventBus
import me.vripper.event.LoadingTasks
import org.koin.core.component.KoinComponent
@@ -13,14 +9,11 @@ internal object Tasks : KoinComponent {
private val eventBus: EventBus by inject()
private var current = 0
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
@Synchronized
fun increment() {
if (current == 0) {
coroutineScope.launch {
eventBus.publishEvent(LoadingTasks(true))
}
eventBus.publishEvent(LoadingTasks(true))
}
current += 1
}
@@ -29,9 +22,7 @@ internal object Tasks : KoinComponent {
fun decrement() {
current -= 1
if (current == 0) {
coroutineScope.launch {
eventBus.publishEvent(LoadingTasks(false))
}
eventBus.publishEvent(LoadingTasks(false))
}
}
}
@@ -1,14 +1,13 @@
package me.vripper.tasks
import kotlinx.coroutines.launch
import me.vripper.entities.ThreadEntity
import me.vripper.model.Settings
import me.vripper.model.ThreadPostId
import me.vripper.services.DataTransaction
import me.vripper.services.SettingsService
import me.vripper.services.ThreadCacheService
import me.vripper.utilities.GlobalScopeCoroutine
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.executorService
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
@@ -34,15 +33,14 @@ internal class ThreadLookupTask(private val threadId: Long, private val settings
}
if (threadLookupResult.postItemList.size <= settings.downloadSettings.autoQueueThreshold) {
GlobalScopeCoroutine.launch {
executorService.submit(
AddPostTask(threadLookupResult.postItemList.map {
ThreadPostId(
it.threadId, it.postId
)
}).run()
}
})
)
} else {
try {
dataTransaction.save(
ThreadEntity(
title = threadLookupResult.title,
@@ -51,9 +49,6 @@ internal class ThreadLookupTask(private val threadId: Long, private val settings
total = threadLookupResult.postItemList.size
)
)
} catch (e: Exception) {
e.printStackTrace()
}
}
}
} catch (e: Exception) {
@@ -35,7 +35,7 @@ object ApplicationProperties {
fun latestVersion(): String {
val request = HttpRequest.newBuilder()
.uri(URI.create("https://api.github.com/repos/death-claw/vripper-project/releases/latest")).build()
.uri(URI.create("https://api.github.com/repos/dev-claw/vripper-project/releases/latest")).build()
return HttpClient.newHttpClient().use {
val response = it.send(request, HttpResponse.BodyHandlers.ofString())
if (response.statusCode() / 100 != 2) {
@@ -1,14 +0,0 @@
package me.vripper.utilities
import kotlinx.coroutines.CoroutineExceptionHandler
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
private val log by LoggerDelegate()
val errorHandler = CoroutineExceptionHandler { _, exception ->
log.error("Unexpected error", exception)
}
val GlobalScopeCoroutine = CoroutineScope(SupervisorJob() + Dispatchers.IO + errorHandler)
@@ -0,0 +1,6 @@
package me.vripper.utilities
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
val executorService: ExecutorService = Executors.newVirtualThreadPerTaskExecutor()
@@ -1,5 +1,7 @@
package me.vripper.vgapi
import dev.failsafe.Failsafe
import dev.failsafe.function.CheckedSupplier
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.withPermit
import me.vripper.exception.DownloadException
@@ -11,8 +13,6 @@ import me.vripper.services.VGAuthService
import me.vripper.tasks.Tasks
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.RequestLimit
import net.jodah.failsafe.Failsafe
import net.jodah.failsafe.function.CheckedSupplier
import org.apache.hc.client5.http.classic.methods.HttpGet
import org.apache.hc.core5.http.io.entity.EntityUtils
import org.apache.hc.core5.net.URIBuilder
@@ -42,9 +42,9 @@ internal class PostLookupAPIParser(private val threadId: Long, private val postI
log.debug("Requesting {}", httpGet)
Tasks.increment()
return try {
Failsafe.with(retryPolicyService.buildGenericRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
Failsafe.with(retryPolicyService.buildRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
log.error(
"Failed to process thread $threadId, post $postId", it.failure
"Failed to process thread $threadId, post $postId", it.exception
)
}.get(CheckedSupplier {
runBlocking {
@@ -1,5 +1,7 @@
package me.vripper.vgapi
import dev.failsafe.Failsafe
import dev.failsafe.function.CheckedSupplier
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.withPermit
import me.vripper.exception.DownloadException
@@ -11,8 +13,6 @@ import me.vripper.services.VGAuthService
import me.vripper.tasks.Tasks
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.RequestLimit
import net.jodah.failsafe.Failsafe
import net.jodah.failsafe.function.CheckedSupplier
import org.apache.hc.client5.http.classic.methods.HttpGet
import org.apache.hc.core5.http.io.entity.EntityUtils
import org.apache.hc.core5.net.URIBuilder
@@ -43,10 +43,10 @@ internal class ThreadLookupAPIParser(private val threadId: Long) : KoinComponent
log.debug("Requesting {}", httpGet)
Tasks.increment()
return try {
Failsafe.with(retryPolicyService.buildGenericRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
Failsafe.with(retryPolicyService.buildRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
log.error(
"Failed to process thread $threadId",
it.failure
it.exception
)
}.get(CheckedSupplier {
runBlocking {
@@ -49,8 +49,8 @@ class VripperGuiApplication : App(
with(stage) {
width = widgetsController.currentSettings.width
height = widgetsController.currentSettings.height
minWidth = 800.0
minHeight = 600.0
minWidth = 100.0
minHeight = 100.0
icons.addAll(
listOf(
Image("icons/16x16.png"),
@@ -23,8 +23,7 @@ class AboutFragment : Fragment("About") {
padding = Insets(15.0, 15.0, 15.0, 15.0)
spacing = 15.0
imageview("icons/64x64.png")
vbox {
spacing = 5.0
vbox(spacing = 5.0) {
text("VRipper") {
style {
fontWeight = FontWeight.BOLD
@@ -32,15 +31,29 @@ class AboutFragment : Fragment("About") {
}
}
text("Version ${ApplicationProperties.VERSION}")
text("Developed by death-claw and VRipper working group")
hyperlink("Home Page") {
action {
openLink("https://github.com/death-claw/vripper-project")
text("Developed by dev-claw and VRipper working group")
hbox(spacing = 5.0) {
hyperlink {
imageview("icons/github-mark.png").apply {
isPreserveRatio = true
fitHeight = 32.0
}
action {
openLink("https://github.com/dev-claw/vripper-project")
}
}
hyperlink {
imageview("icons/buymeacoffee-logo.png").apply {
isPreserveRatio = true
fitHeight = 32.0
}
action {
openLink("https://buymeacoffee.com/devclaw")
}
}
}
}
}
}
tab("System") {
form {
@@ -1,19 +0,0 @@
package me.vripper.gui.components.fragments
import javafx.beans.property.SimpleBooleanProperty
import javafx.collections.FXCollections
import tornadofx.Fragment
import tornadofx.listview
import tornadofx.useCheckbox
class ColumnSelectionFragment : Fragment("Column Selection") {
val map: MutableMap<String, SimpleBooleanProperty> by param()
override val root = listview<String> {
items = FXCollections.observableArrayList(map.keys)
useCheckbox { listItem ->
map[listItem]!!
}
}
}
@@ -1,10 +1,13 @@
package me.vripper.gui.components.fragments
import atlantafx.base.theme.Styles
import javafx.beans.property.SimpleStringProperty
import javafx.geometry.Pos
import javafx.scene.control.ComboBox
import kotlinx.coroutines.*
import me.vripper.gui.controller.PostController
import org.kordamp.ikonli.feather.Feather
import org.kordamp.ikonli.javafx.FontIcon
import tornadofx.*
class RenameFragment : Fragment("Rename download post") {
@@ -35,6 +38,8 @@ class RenameFragment : Fragment("Rename download post") {
}
}
button("Rename") {
graphic = FontIcon.of(Feather.EDIT)
addClass(Styles.ACCENT)
disableWhen(comboBox.editor.textProperty().isEmpty)
action {
coroutineScope.launch {
@@ -31,8 +31,9 @@ class SettingsFragment : Fragment("Settings") {
tabpane {
tabClosingPolicy = TabPane.TabClosingPolicy.UNAVAILABLE
VBox.setVgrow(this, Priority.ALWAYS)
minWidth = 700.0
minHeight = 400.0
minWidth = 100.0
minHeight = 100.0
prefHeight = 400.0
tab(downloadSettingsFragment.title) {
add(downloadSettingsFragment)
graphic = FontIcon.of(Feather.FOLDER)
@@ -60,7 +60,10 @@ class ActionBarView : View() {
action {
find<AddLinksFragment>().apply {
input.clear()
}.openModal()
}.openModal()?.apply {
minWidth = 100.0
minHeight = 100.0
}
}
}
separator(Orientation.VERTICAL)
@@ -111,7 +114,10 @@ class ActionBarView : View() {
contentDisplay = ContentDisplay.GRAPHIC_ONLY
tooltip("Open settings menu [Ctrl+P]")
action {
find<SettingsFragment>().openModal(owner = primaryStage)
find<SettingsFragment>().openModal()?.apply {
minWidth = 100.0
minHeight = 100.0
}
}
}
separator(Orientation.VERTICAL)
@@ -273,7 +273,9 @@ class ImagesTableView : View("Photos") {
fun setPostId(postId: Long?) {
ActiveUICoroutines.images.forEach { it.cancel() }
ActiveUICoroutines.images.clear()
items.clear()
runLater {
items.clear()
}
if (postId == null) {
return
}
@@ -51,10 +51,12 @@ class LoadingView : View("VRipper") {
if (!grpcEndpointService.ready()) {
val sessionView = find<SessionFragment>()
runLater {
sessionView.openModal().also {
it?.setOnCloseRequest {
sessionView.openModal()?.apply {
setOnCloseRequest {
VripperGuiApplication.APP_INSTANCE.stop()
}
minWidth = 100.0
minHeight = 100.0
}
}
}
@@ -228,8 +228,8 @@ class LogTableView : View() {
private fun openLog(item: LogModel) {
find<LogMessageFragment>(mapOf(LogMessageFragment::logModel to item)).openModal()?.apply {
minWidth = 600.0
minHeight = 400.0
minWidth = 100.0
minHeight = 100.0
}
}
}
@@ -67,7 +67,10 @@ class MenuBarView : View() {
action {
find<AddLinksFragment>().apply {
input.clear()
}.openModal()
}.openModal()?.apply {
minWidth = 100.0
minHeight = 100.0
}
}
}
separator()
@@ -113,14 +116,20 @@ class MenuBarView : View() {
item("Settings", KeyCodeCombination(KeyCode.P, KeyCombination.CONTROL_DOWN)).apply {
graphic = FontIcon.of(Feather.SETTINGS)
action {
find<SettingsFragment>().openModal(owner = primaryStage)
find<SettingsFragment>().openModal()?.apply {
minWidth = 100.0
minHeight = 100.0
}
}
}
separator()
item("Change session", KeyCodeCombination(KeyCode.S, KeyCombination.SHIFT_DOWN)) {
graphic = FontIcon.of(Feather.LINK_2)
action {
find<SessionFragment>().openModal()
find<SessionFragment>().openModal()?.apply {
minWidth = 100.0
minHeight = 100.0
}
}
}
separator()
@@ -188,7 +197,7 @@ class MenuBarView : View() {
owner = primaryStage,
) {
if (it == ButtonType.YES) {
openLink("https://github.com/death-claw/vripper-project/releases/tag/$latestVersion")
openLink("https://github.com/dev-claw/vripper-project/releases/tag/$latestVersion")
}
}
} else {
@@ -206,8 +215,8 @@ class MenuBarView : View() {
graphic = FontIcon.of(Feather.INFO)
action {
find<AboutFragment>().openModal()?.apply {
this.minWidth = 625.0
this.minHeight = 200.0
this.minWidth = 100.0
this.minHeight = 100.0
}
}
}
@@ -97,7 +97,7 @@ class PostsTableView : View() {
isTableMenuButtonVisible = true
primaryStage.addEventFilter(KeyEvent.KEY_PRESSED) { event ->
if (event.code == KeyCode.DELETE) {
if (isCurrentTab() && selectionModel.selectedItems.isNotEmpty()) {
if (isCurrentTab() && selectionModel.selectedItems.isNotEmpty() && this.isFocused) {
deleteSelected()
}
}
@@ -178,7 +178,10 @@ class PostsTableView : View() {
action {
find<AddLinksFragment>().apply {
input.clear()
}.openModal()
}.openModal()?.apply {
minWidth = 100.0
minHeight = 100.0
}
}
})
column("Preview", PostModel::previewListProperty) {
@@ -414,7 +417,7 @@ class PostsTableView : View() {
coroutineScope.launch {
postController.onNewPosts().collect {
runLater {
tableView.items.addAll(it)
items.addAll(it)
tableView.sort()
}
}
@@ -443,7 +446,7 @@ class PostsTableView : View() {
coroutineScope.launch {
postController.onDeletePosts().collect {
runLater {
items.removeIf { p -> p.postId == it }
items.items.removeIf { p -> p.postId == it }
tableView.sort()
}
}
@@ -469,7 +472,8 @@ class PostsTableView : View() {
RenameFragment::altTitles to post.altTitles
)
).openModal()?.apply {
minWidth = 450.0
minWidth = 100.0
minHeight = 100.0
}
}
@@ -500,7 +504,7 @@ class PostsTableView : View() {
coroutineScope.launch {
postController.delete(postIdList)
runLater {
tableView.items.removeIf { postIdList.contains(it.postId) }
items.items.removeIf { postIdList.contains(it.postId) }
}
}
}
@@ -65,7 +65,7 @@ class ThreadTableView : View() {
isTableMenuButtonVisible = true
primaryStage.addEventFilter(KeyEvent.KEY_PRESSED) { event ->
if (event.code == KeyCode.DELETE) {
if (isCurrentTab() && selectionModel.selectedItems.isNotEmpty()) {
if (isCurrentTab() && selectionModel.selectedItems.isNotEmpty() && this.isFocused) {
deleteSelected()
}
}
@@ -248,10 +248,8 @@ class ThreadTableView : View() {
private fun selectPosts(threadId: Long) {
find<ThreadSelectionTableFragment>(mapOf(ThreadSelectionTableFragment::threadId to threadId)).openModal()
?.apply {
minWidth = 600.0
minHeight = 400.0
width = 800.0
height = 600.0
minWidth = 100.0
minHeight = 100.0
}
}
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.2 KiB