mirror of
https://github.com/dev-claw/vripper-project.git
synced 2026-08-19 08:35:41 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
804c3d0c04 |
@@ -25,7 +25,7 @@
|
||||
<koin.version>4.0.0</koin.version>
|
||||
<h2.version>2.2.224</h2.version>
|
||||
<htmlcleaner.version>2.29</htmlcleaner.version>
|
||||
<failsafe.version>2.4.4</failsafe.version>
|
||||
<failsafe.version>3.3.2</failsafe.version>
|
||||
<caffeine.version>3.1.8</caffeine.version>
|
||||
<jna-platform.version>5.14.0</jna-platform.version>
|
||||
<kotlinx-serialization-json.version>1.7.3</kotlinx-serialization-json.version>
|
||||
@@ -142,8 +142,8 @@
|
||||
<version>${htmlcleaner.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>dev.failsafe</groupId>
|
||||
<artifactId>failsafe</artifactId>
|
||||
<groupId>net.jodah</groupId>
|
||||
<version>${failsafe.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
|
||||
@@ -69,7 +69,7 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<artifactId>failsafe</artifactId>
|
||||
<groupId>net.jodah</groupId>
|
||||
<groupId>dev.failsafe</groupId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<artifactId>caffeine</artifactId>
|
||||
|
||||
@@ -2,11 +2,11 @@ package me.vripper
|
||||
|
||||
import me.vripper.data.repositories.ImageRepository
|
||||
import me.vripper.data.repositories.MetadataRepository
|
||||
import me.vripper.data.repositories.PostDownloadStateRepository
|
||||
import me.vripper.data.repositories.PostRepository
|
||||
import me.vripper.data.repositories.ThreadRepository
|
||||
import me.vripper.data.repositories.impl.ImageRepositoryImpl
|
||||
import me.vripper.data.repositories.impl.MetadataRepositoryImpl
|
||||
import me.vripper.data.repositories.impl.PostDownloadStateRepositoryImpl
|
||||
import me.vripper.data.repositories.impl.PostRepositoryImpl
|
||||
import me.vripper.data.repositories.impl.ThreadRepositoryImpl
|
||||
import me.vripper.download.DownloadService
|
||||
import me.vripper.event.EventBus
|
||||
@@ -29,8 +29,8 @@ val coreModule = module {
|
||||
single<ImageRepository> {
|
||||
ImageRepositoryImpl()
|
||||
}
|
||||
single<PostDownloadStateRepository> {
|
||||
PostDownloadStateRepositoryImpl()
|
||||
single<PostRepository> {
|
||||
PostRepositoryImpl()
|
||||
}
|
||||
single<MetadataRepository> {
|
||||
MetadataRepositoryImpl()
|
||||
|
||||
+3
-4
@@ -1,12 +1,11 @@
|
||||
package me.vripper.data.repositories
|
||||
|
||||
import me.vripper.entities.PostEntity
|
||||
import java.util.*
|
||||
|
||||
internal interface PostDownloadStateRepository {
|
||||
internal interface PostRepository {
|
||||
fun save(postEntities: List<PostEntity>): List<PostEntity>
|
||||
fun findByPostId(postId: Long): Optional<PostEntity>
|
||||
fun findById(id: Long): Optional<PostEntity>
|
||||
fun findByPostId(postId: Long): PostEntity?
|
||||
fun findById(id: Long): PostEntity?
|
||||
fun findCompleted(): List<Long>
|
||||
fun findAll(): List<PostEntity>
|
||||
fun existByPostId(postId: Long): Boolean
|
||||
+7
-17
@@ -1,6 +1,6 @@
|
||||
package me.vripper.data.repositories.impl
|
||||
|
||||
import me.vripper.data.repositories.PostDownloadStateRepository
|
||||
import me.vripper.data.repositories.PostRepository
|
||||
import me.vripper.data.tables.PostTable
|
||||
import me.vripper.entities.PostEntity
|
||||
import me.vripper.entities.Status
|
||||
@@ -8,10 +8,9 @@ import org.jetbrains.exposed.sql.*
|
||||
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
|
||||
import org.jetbrains.exposed.sql.transactions.TransactionManager
|
||||
import java.sql.Connection
|
||||
import java.util.*
|
||||
|
||||
internal class PostDownloadStateRepositoryImpl :
|
||||
PostDownloadStateRepository {
|
||||
internal class PostRepositoryImpl :
|
||||
PostRepository {
|
||||
|
||||
private val delimiter = ";"
|
||||
|
||||
@@ -37,15 +36,11 @@ internal class PostDownloadStateRepositoryImpl :
|
||||
}.map(::transform)
|
||||
}
|
||||
|
||||
override fun findByPostId(postId: Long): Optional<PostEntity> {
|
||||
override fun findByPostId(postId: Long): PostEntity? {
|
||||
val result = PostTable.selectAll().where {
|
||||
PostTable.postId eq postId
|
||||
}.map(::transform)
|
||||
return if (result.isEmpty()) {
|
||||
Optional.empty()
|
||||
} else {
|
||||
Optional.of(result.first())
|
||||
}
|
||||
return result.firstOrNull()
|
||||
}
|
||||
|
||||
override fun findCompleted(): List<Long> {
|
||||
@@ -54,16 +49,11 @@ internal class PostDownloadStateRepositoryImpl :
|
||||
}.map { it[PostTable.postId] }
|
||||
}
|
||||
|
||||
override fun findById(id: Long): Optional<PostEntity> {
|
||||
override fun findById(id: Long): PostEntity? {
|
||||
val result = PostTable.selectAll().where {
|
||||
PostTable.id eq id
|
||||
}.map { transform(it) }
|
||||
|
||||
return if (result.isEmpty()) {
|
||||
Optional.empty()
|
||||
} else {
|
||||
Optional.of(result.first())
|
||||
}
|
||||
return result.firstOrNull()
|
||||
}
|
||||
|
||||
override fun findAll(): List<PostEntity> {
|
||||
@@ -1,7 +1,8 @@
|
||||
package me.vripper.download
|
||||
|
||||
import dev.failsafe.Failsafe
|
||||
import dev.failsafe.RetryPolicy
|
||||
import kotlinx.coroutines.Runnable
|
||||
import kotlinx.coroutines.launch
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.entities.PostEntity
|
||||
import me.vripper.entities.Status
|
||||
@@ -16,10 +17,8 @@ import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.RetryPolicyService
|
||||
import me.vripper.services.SettingsService
|
||||
import me.vripper.services.VGAuthService
|
||||
import me.vripper.utilities.GlobalScopeCoroutine
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import net.jodah.failsafe.Failsafe
|
||||
import net.jodah.failsafe.RetryPolicy
|
||||
import me.vripper.utilities.executorService
|
||||
import org.jetbrains.exposed.sql.transactions.transaction
|
||||
import java.util.concurrent.locks.ReentrantLock
|
||||
import kotlin.concurrent.withLock
|
||||
@@ -46,14 +45,14 @@ internal class DownloadService(
|
||||
lock.withLock {
|
||||
candidates.addAll(getCandidates(candidateCount()))
|
||||
candidates.forEach {
|
||||
if (canRun(it.context.imageEntity.host)) {
|
||||
if (canRun(it.imageEntity.host)) {
|
||||
accepted.add(it)
|
||||
running[it.context.imageEntity.host]!!.add(it)
|
||||
log.debug("${it.context.imageEntity.url} accepted to run")
|
||||
running[it.imageEntity.host]!!.add(it)
|
||||
log.debug("${it.imageEntity.url} accepted to run")
|
||||
}
|
||||
}
|
||||
accepted.forEach {
|
||||
pending[it.context.imageEntity.host]?.remove(it)
|
||||
pending[it.imageEntity.host]?.remove(it)
|
||||
scheduleForDownload(it)
|
||||
}
|
||||
accepted.clear()
|
||||
@@ -144,18 +143,18 @@ internal class DownloadService(
|
||||
}
|
||||
|
||||
private fun isPending(postId: Long): Boolean {
|
||||
return pending.values.flatten().any { it.context.imageEntity.postId == postId }
|
||||
return pending.values.flatten().any { it.imageEntity.postId == postId }
|
||||
}
|
||||
|
||||
private fun isRunning(postId: Long): Boolean {
|
||||
return running.values.flatten().any { it.context.imageEntity.postId == postId }
|
||||
return running.values.flatten().any { it.imageEntity.postId == postId }
|
||||
}
|
||||
|
||||
private fun stopAll() {
|
||||
lock.withLock {
|
||||
pending.values.clear()
|
||||
running.values.flatten().forEach { obj: ImageDownloadRunnable -> obj.stop() }
|
||||
while (running.values.flatten().count { !it.context.completed } > 0) {
|
||||
while (running.values.flatten().count { !it.completed } > 0) {
|
||||
Thread.sleep(100)
|
||||
}
|
||||
dataTransaction.findAllNonCompletedPostIds().forEach {
|
||||
@@ -169,13 +168,13 @@ internal class DownloadService(
|
||||
lock.withLock {
|
||||
for (postId in postIds) {
|
||||
pending.values.forEach { pending ->
|
||||
pending.removeIf { it.context.imageEntity.postId == postId }
|
||||
pending.removeIf { it.imageEntity.postId == postId }
|
||||
}
|
||||
running.values.flatten()
|
||||
.filter { p: ImageDownloadRunnable -> p.context.imageEntity.postId == postId }
|
||||
.filter { p: ImageDownloadRunnable -> p.imageEntity.postId == postId }
|
||||
.forEach { obj: ImageDownloadRunnable -> obj.stop() }
|
||||
while (running.values.flatten()
|
||||
.count { !it.context.completed && it.context.imageEntity.postId == postId } > 0
|
||||
.count { !it.completed && it.imageEntity.postId == postId } > 0
|
||||
) {
|
||||
Thread.sleep(100)
|
||||
}
|
||||
@@ -213,7 +212,7 @@ internal class DownloadService(
|
||||
|
||||
val list: List<ImageDownloadRunnable> =
|
||||
pending[host]!!.sortedWith(Comparator.comparingInt<ImageDownloadRunnable> { it.postRank }
|
||||
.thenComparingInt { it.context.imageEntity.index })
|
||||
.thenComparingInt { it.imageEntity.index })
|
||||
|
||||
for (imageDownloadRunnable in list) {
|
||||
val count = hostIntegerMap[host] ?: 0
|
||||
@@ -229,47 +228,42 @@ internal class DownloadService(
|
||||
}
|
||||
|
||||
private fun scheduleForDownload(imageDownloadRunnable: ImageDownloadRunnable) {
|
||||
log.debug("Scheduling a job for ${imageDownloadRunnable.context.imageEntity.url}")
|
||||
GlobalScopeCoroutine.launch {
|
||||
eventBus.publishEvent(QueueStateEvent(QueueState(runningCount(), pendingCount())))
|
||||
try {
|
||||
Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicyForDownload("Failed to download ${imageDownloadRunnable.context.imageEntity.url}: "))
|
||||
.onFailure {
|
||||
log.error(
|
||||
"Failed to download ${imageDownloadRunnable.context.imageEntity.url} after ${it.attemptCount} tries",
|
||||
it.failure
|
||||
)
|
||||
val image = imageDownloadRunnable.context.imageEntity
|
||||
image.status = Status.ERROR
|
||||
dataTransaction.updateImage(image)
|
||||
}
|
||||
.onComplete {
|
||||
afterJobFinish(imageDownloadRunnable)
|
||||
eventBus.publishEvent(
|
||||
QueueStateEvent(
|
||||
QueueState(
|
||||
runningCount(), pendingCount()
|
||||
)
|
||||
)
|
||||
)
|
||||
eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataTransaction.countImagesInError())))
|
||||
log.debug(
|
||||
"Finished downloading ${imageDownloadRunnable.context.imageEntity.url}"
|
||||
)
|
||||
}.run(imageDownloadRunnable::run)
|
||||
} catch (e: Exception) {
|
||||
log.error("Download Failure", e)
|
||||
log.debug("Scheduling a job for ${imageDownloadRunnable.imageEntity.url}")
|
||||
eventBus.publishEvent(QueueStateEvent(QueueState(runningCount(), pendingCount())))
|
||||
Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicy("Failed to download ${imageDownloadRunnable.imageEntity.url}: "))
|
||||
.with(executorService)
|
||||
.onFailure {
|
||||
log.error(
|
||||
"Failed to download ${imageDownloadRunnable.imageEntity.url} after ${it.attemptCount} tries",
|
||||
it.exception
|
||||
)
|
||||
val image = imageDownloadRunnable.imageEntity
|
||||
image.status = Status.ERROR
|
||||
dataTransaction.updateImage(image)
|
||||
}
|
||||
}
|
||||
.onComplete {
|
||||
afterJobFinish(imageDownloadRunnable)
|
||||
eventBus.publishEvent(
|
||||
QueueStateEvent(
|
||||
QueueState(
|
||||
runningCount(), pendingCount()
|
||||
)
|
||||
)
|
||||
)
|
||||
eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataTransaction.countImagesInError())))
|
||||
log.debug(
|
||||
"Finished downloading ${imageDownloadRunnable.imageEntity.url}"
|
||||
)
|
||||
}.runAsync(imageDownloadRunnable)
|
||||
}
|
||||
|
||||
private fun afterJobFinish(imageDownloadRunnable: ImageDownloadRunnable) {
|
||||
lock.withLock {
|
||||
val image = imageDownloadRunnable.context.imageEntity
|
||||
val image = imageDownloadRunnable.imageEntity
|
||||
running[image.host]!!.remove(imageDownloadRunnable)
|
||||
if (!isPending(image.postId) && !isRunning(
|
||||
image.postId
|
||||
) && !imageDownloadRunnable.context.stopped
|
||||
) && !imageDownloadRunnable.stopped
|
||||
) {
|
||||
dataTransaction.finishPost(image.postId, true)
|
||||
}
|
||||
|
||||
@@ -15,8 +15,6 @@ internal class ImageDownloadContext(val imageEntity: ImageEntity, val settings:
|
||||
HttpClientContext.create().apply { cookieStore = BasicCookieStore() }
|
||||
val requests = mutableListOf<HttpUriRequestBase>()
|
||||
val postId = imageEntity.postIdRef
|
||||
var stopped = false
|
||||
var completed = false
|
||||
|
||||
fun cancelCoroutines() {
|
||||
runBlocking {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package me.vripper.download
|
||||
|
||||
import dev.failsafe.function.CheckedRunnable
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.entities.Status
|
||||
import me.vripper.exception.DownloadException
|
||||
@@ -13,7 +14,6 @@ import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.PathUtils.getExtension
|
||||
import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
|
||||
import me.vripper.utilities.PathUtils.sanitize
|
||||
import net.jodah.failsafe.function.CheckedRunnable
|
||||
import org.koin.core.component.KoinComponent
|
||||
import org.koin.core.component.inject
|
||||
import java.io.IOException
|
||||
@@ -26,23 +26,23 @@ import kotlin.io.path.Path
|
||||
import kotlin.io.path.pathString
|
||||
|
||||
internal class ImageDownloadRunnable(
|
||||
private val imageEntity: ImageEntity, val postRank: Int, private val settings: Settings
|
||||
val imageEntity: ImageEntity, val postRank: Int, private val settings: Settings
|
||||
) : KoinComponent, CheckedRunnable {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
private val dataTransaction: DataTransaction by inject()
|
||||
private val hosts: List<Host> = getKoin().getAll()
|
||||
var completed = false
|
||||
var stopped = false
|
||||
|
||||
val context: ImageDownloadContext = ImageDownloadContext(imageEntity, settings)
|
||||
private lateinit var context: ImageDownloadContext
|
||||
|
||||
@Throws(DownloadException::class)
|
||||
fun download() {
|
||||
try {
|
||||
imageEntity.status = Status.DOWNLOADING
|
||||
imageEntity.downloaded = 0
|
||||
dataTransaction.updateImage(imageEntity)
|
||||
synchronized(imageEntity.postId.toString().intern()) {
|
||||
val post = dataTransaction.findPostById(context.postId).orElseThrow()
|
||||
val post = dataTransaction.findPostById(context.postId)
|
||||
if (post.status != Status.DOWNLOADING) {
|
||||
post.status = Status.DOWNLOADING
|
||||
dataTransaction.updatePost(post)
|
||||
@@ -54,7 +54,7 @@ internal class ImageDownloadRunnable(
|
||||
log.debug("Resolved name for ${imageEntity.url}: ${downloadedImage.name}")
|
||||
log.debug("Downloaded image {} to {}", imageEntity.url, downloadedImage.path)
|
||||
synchronized(imageEntity.postId.toString().intern()) {
|
||||
val post = dataTransaction.findPostById(context.postId).orElseThrow()
|
||||
val post = dataTransaction.findPostById(context.postId)
|
||||
val downloadDirectory = Path(post.downloadDirectory, post.folderName).pathString
|
||||
checkImageTypeAndRename(
|
||||
downloadDirectory, downloadedImage, imageEntity.index
|
||||
@@ -70,7 +70,7 @@ internal class ImageDownloadRunnable(
|
||||
dataTransaction.updateImage(imageEntity)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
if (context.stopped) {
|
||||
if (stopped) {
|
||||
return
|
||||
}
|
||||
imageEntity.status = Status.ERROR
|
||||
@@ -119,19 +119,26 @@ internal class ImageDownloadRunnable(
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(Exception::class)
|
||||
override fun run() {
|
||||
context = ImageDownloadContext(imageEntity, settings)
|
||||
try {
|
||||
if (context.stopped) {
|
||||
if (stopped) {
|
||||
return
|
||||
}
|
||||
download()
|
||||
} finally {
|
||||
context.completed = true
|
||||
completed = true
|
||||
context.cancelCoroutines()
|
||||
}
|
||||
}
|
||||
|
||||
fun stop() {
|
||||
stopped = true
|
||||
context.requests.forEach { it.abort() }
|
||||
context.cancelCoroutines()
|
||||
dataTransaction.updateImage(context.imageEntity)
|
||||
}
|
||||
|
||||
override fun equals(other: Any?): Boolean {
|
||||
if (this === other) return true
|
||||
if (other == null || javaClass != other.javaClass) return false
|
||||
@@ -142,11 +149,4 @@ internal class ImageDownloadRunnable(
|
||||
override fun hashCode(): Int {
|
||||
return Objects.hash(imageEntity.id)
|
||||
}
|
||||
|
||||
fun stop() {
|
||||
context.requests.forEach { it.abort() }
|
||||
context.cancelCoroutines()
|
||||
context.stopped = true
|
||||
dataTransaction.updateImage(context.imageEntity)
|
||||
}
|
||||
}
|
||||
@@ -106,7 +106,7 @@ internal abstract class Host(
|
||||
return BufferedOutputStream(Files.newOutputStream(tempImage)).use { bos ->
|
||||
val image = context.imageEntity
|
||||
synchronized(image.postId.toString().intern()) {
|
||||
val post = dataTransaction.findPostById(context.postId).orElseThrow()
|
||||
val post = dataTransaction.findPostById(context.postId)
|
||||
val size = if (image.size < 0) {
|
||||
response.entity.contentLength
|
||||
} else {
|
||||
@@ -134,7 +134,7 @@ internal abstract class Host(
|
||||
}
|
||||
}
|
||||
while (response.entity.content.read(buffer)
|
||||
.also { read = it } != -1 && !context.stopped
|
||||
.also { read = it } != -1
|
||||
) {
|
||||
bos.write(buffer, 0, read)
|
||||
image.downloaded += read
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package me.vripper.services
|
||||
|
||||
import kotlinx.coroutines.flow.*
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.time.sample
|
||||
import kotlinx.serialization.json.Json
|
||||
import me.vripper.download.DownloadService
|
||||
@@ -13,9 +12,9 @@ import me.vripper.tasks.AddPostTask
|
||||
import me.vripper.tasks.ThreadLookupTask
|
||||
import me.vripper.utilities.ApplicationProperties
|
||||
import me.vripper.utilities.ApplicationProperties.VRIPPER_DIR
|
||||
import me.vripper.utilities.GlobalScopeCoroutine
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.PathUtils
|
||||
import me.vripper.utilities.executorService
|
||||
import org.h2.jdbc.JdbcSQLNonTransientConnectionException
|
||||
import java.sql.DriverManager
|
||||
import java.time.Duration
|
||||
@@ -59,17 +58,17 @@ internal class AppEndpointService(
|
||||
threadId = m.group(1).toLong()
|
||||
postId = m.group(4)?.toLong()
|
||||
if (postId == null) {
|
||||
GlobalScopeCoroutine.launch {
|
||||
executorService.submit(
|
||||
ThreadLookupTask(
|
||||
threadId, settingsService.settings
|
||||
).run()
|
||||
}
|
||||
)
|
||||
)
|
||||
} else {
|
||||
GlobalScopeCoroutine.launch {
|
||||
executorService.submit(
|
||||
AddPostTask(
|
||||
listOf(ThreadPostId(threadId, postId))
|
||||
).run()
|
||||
}
|
||||
)
|
||||
)
|
||||
}
|
||||
} else {
|
||||
log.error("Invalid link $link, link is missing the threadId")
|
||||
@@ -81,15 +80,13 @@ internal class AppEndpointService(
|
||||
|
||||
override suspend fun restartAll(posIds: List<Long>) {
|
||||
lock.withLock {
|
||||
downloadService.restartAll(posIds.map { dataTransaction.findPostByPostId(it) }.filter { it.isPresent }
|
||||
.map { it.get() })
|
||||
downloadService.restartAll(posIds.filter { dataTransaction.exists(it) }
|
||||
.map { dataTransaction.findPostByPostId(it) })
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun download(posts: List<ThreadPostId>) {
|
||||
GlobalScopeCoroutine.launch {
|
||||
AddPostTask(posts).run()
|
||||
}
|
||||
executorService.submit(AddPostTask(posts))
|
||||
}
|
||||
|
||||
override suspend fun stopAll(postIdList: List<Long>) {
|
||||
@@ -185,16 +182,21 @@ internal class AppEndpointService(
|
||||
}
|
||||
|
||||
override suspend fun rename(postId: Long, newName: String) {
|
||||
GlobalScopeCoroutine.launch {
|
||||
executorService.submit {
|
||||
synchronized(postId.toString().intern()) {
|
||||
dataTransaction.findPostByPostId(postId).ifPresent { post ->
|
||||
if (Path(post.downloadDirectory, post.folderName).exists()) {
|
||||
PathUtils.rename(
|
||||
dataTransaction.findImagesByPostId(postId), post.downloadDirectory, post.folderName, newName
|
||||
)
|
||||
if (dataTransaction.exists(postId)) {
|
||||
dataTransaction.findPostByPostId(postId).let { post ->
|
||||
if (Path(post.downloadDirectory, post.folderName).exists()) {
|
||||
PathUtils.rename(
|
||||
dataTransaction.findImagesByPostId(postId),
|
||||
post.downloadDirectory,
|
||||
post.folderName,
|
||||
newName
|
||||
)
|
||||
}
|
||||
post.folderName = PathUtils.sanitize(newName)
|
||||
dataTransaction.updatePost(post)
|
||||
}
|
||||
post.folderName = PathUtils.sanitize(newName)
|
||||
dataTransaction.updatePost(post)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -251,7 +253,7 @@ internal class AppEndpointService(
|
||||
}
|
||||
|
||||
override suspend fun findPost(postId: Long): Post {
|
||||
return mapper(dataTransaction.findPostByPostId(postId).orElseThrow())
|
||||
return mapper(dataTransaction.findPostByPostId(postId))
|
||||
}
|
||||
|
||||
override suspend fun findImagesByPostId(postId: Long): List<Image> {
|
||||
@@ -353,7 +355,7 @@ internal class AppEndpointService(
|
||||
val addedAt = it.getTimestamp("ADDED_AT")
|
||||
val folderName = it.getString("FOLDER_NAME") ?: ""
|
||||
|
||||
val exists = dataTransaction.findPostByPostId(postId).isPresent
|
||||
val exists = dataTransaction.exists(postId)
|
||||
if (exists) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package me.vripper.services
|
||||
|
||||
import com.github.benmanes.caffeine.cache.Caffeine
|
||||
import com.github.benmanes.caffeine.cache.LoadingCache
|
||||
import me.vripper.data.repositories.ImageRepository
|
||||
import me.vripper.data.repositories.MetadataRepository
|
||||
import me.vripper.data.repositories.PostDownloadStateRepository
|
||||
import me.vripper.data.repositories.PostRepository
|
||||
import me.vripper.data.repositories.ThreadRepository
|
||||
import me.vripper.entities.*
|
||||
import me.vripper.event.*
|
||||
@@ -12,12 +14,13 @@ import me.vripper.utilities.PathUtils.sanitize
|
||||
import me.vripper.vgapi.PostItem
|
||||
import org.jetbrains.exposed.sql.transactions.transaction
|
||||
import java.util.*
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import kotlin.io.path.pathString
|
||||
|
||||
internal class DataTransaction(
|
||||
private val settingsService: SettingsService,
|
||||
private val postDownloadStateRepository: PostDownloadStateRepository,
|
||||
private val postRepository: PostRepository,
|
||||
private val imageRepository: ImageRepository,
|
||||
private val threadRepository: ThreadRepository,
|
||||
private val metadataRepository: MetadataRepository,
|
||||
@@ -25,15 +28,24 @@ internal class DataTransaction(
|
||||
) {
|
||||
|
||||
private val nextRank = AtomicInteger(transaction { getQueuePosition() }?.plus(1) ?: 0)
|
||||
private val postEntityIdCache: LoadingCache<Long, PostEntity> =
|
||||
Caffeine.newBuilder().expireAfterAccess(5, TimeUnit.MINUTES).build { id ->
|
||||
transaction { postRepository.findById(id) }
|
||||
}
|
||||
|
||||
private val postPostIdCache: LoadingCache<Long, PostEntity> =
|
||||
Caffeine.newBuilder().expireAfterAccess(5, TimeUnit.MINUTES).build { id ->
|
||||
transaction { postRepository.findByPostId(id) }
|
||||
}
|
||||
|
||||
private fun save(postEntities: List<PostEntity>): List<PostEntity> {
|
||||
return transaction { postDownloadStateRepository.save(postEntities) }
|
||||
return transaction { postRepository.save(postEntities) }
|
||||
}
|
||||
|
||||
fun saveAndNotify(postEntity: PostEntity, images: List<ImageEntity>) {
|
||||
val savedPost = transaction {
|
||||
val savedPost =
|
||||
postDownloadStateRepository.save(listOf(postEntity.copy(rank = nextRank.andIncrement))).first()
|
||||
postRepository.save(listOf(postEntity.copy(rank = nextRank.andIncrement))).first()
|
||||
save(images.map { it.copy(postIdRef = savedPost.id) })
|
||||
savedPost
|
||||
}
|
||||
@@ -41,12 +53,18 @@ internal class DataTransaction(
|
||||
}
|
||||
|
||||
fun updatePosts(postEntities: List<PostEntity>) {
|
||||
transaction { postDownloadStateRepository.update(postEntities) }
|
||||
transaction { postRepository.update(postEntities) }
|
||||
postEntities.forEach { postEntity ->
|
||||
postPostIdCache.put(postEntity.postId, postEntity)
|
||||
postEntityIdCache.put(postEntity.id, postEntity)
|
||||
}
|
||||
eventBus.publishEvent(PostUpdateEvent(postEntities))
|
||||
}
|
||||
|
||||
fun updatePost(postEntity: PostEntity) {
|
||||
transaction { postDownloadStateRepository.update(postEntity) }
|
||||
transaction { postRepository.update(postEntity) }
|
||||
postPostIdCache.put(postEntity.postId, postEntity)
|
||||
postEntityIdCache.put(postEntity.id, postEntity)
|
||||
eventBus.publishEvent(PostUpdateEvent(listOf(postEntity)))
|
||||
}
|
||||
|
||||
@@ -75,7 +93,10 @@ internal class DataTransaction(
|
||||
}
|
||||
|
||||
fun exists(postId: Long): Boolean {
|
||||
return transaction { postDownloadStateRepository.existByPostId(postId) }
|
||||
if (postPostIdCache.getIfPresent(postId) != null) {
|
||||
return true
|
||||
}
|
||||
return transaction { postRepository.existByPostId(postId) }
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
@@ -128,7 +149,7 @@ internal class DataTransaction(
|
||||
}
|
||||
|
||||
private fun getQueuePosition(): Int? {
|
||||
return postDownloadStateRepository.findMaxRank()
|
||||
return postRepository.findMaxRank()
|
||||
}
|
||||
|
||||
private fun save(imageEntities: List<ImageEntity>) {
|
||||
@@ -136,7 +157,7 @@ internal class DataTransaction(
|
||||
}
|
||||
|
||||
fun finishPost(postId: Long, automatic: Boolean = false) {
|
||||
val post = findPostByPostId(postId).orElseThrow()
|
||||
val post = findPostByPostId(postId)
|
||||
val imagesInErrorStatus = findByPostIdAndIsError(post.postId)
|
||||
if (imagesInErrorStatus.isNotEmpty()) {
|
||||
post.status = Status.ERROR
|
||||
@@ -167,9 +188,13 @@ internal class DataTransaction(
|
||||
transaction {
|
||||
metadataRepository.deleteAllByPostId(postIds)
|
||||
imageRepository.deleteAllByPostId(postIds)
|
||||
postDownloadStateRepository.deleteAll(postIds)
|
||||
postRepository.deleteAll(postIds)
|
||||
sortPostsByRank()
|
||||
}
|
||||
postIds.forEach { postId ->
|
||||
postPostIdCache.get(postId)?.let { postEntityIdCache.invalidate(it.id) }
|
||||
postPostIdCache.invalidate(postId)
|
||||
}
|
||||
eventBus.publishEvent(PostDeleteEvent(postIds = postIds))
|
||||
eventBus.publishEvent(ErrorCountEvent(ErrorCount(countImagesInError())))
|
||||
}
|
||||
@@ -180,7 +205,7 @@ internal class DataTransaction(
|
||||
}
|
||||
|
||||
fun clearCompleted(): List<Long> {
|
||||
val completed = transaction { postDownloadStateRepository.findCompleted() }
|
||||
val completed = transaction { postRepository.findCompleted() }
|
||||
remove(completed)
|
||||
return completed
|
||||
}
|
||||
@@ -225,15 +250,15 @@ internal class DataTransaction(
|
||||
}
|
||||
|
||||
fun setDownloadingToStopped() {
|
||||
transaction { postDownloadStateRepository.setDownloadingToStopped() }
|
||||
transaction { postRepository.setDownloadingToStopped() }
|
||||
}
|
||||
|
||||
fun findAllPosts(): List<PostEntity> {
|
||||
return transaction { postDownloadStateRepository.findAll() }
|
||||
return transaction { postRepository.findAll() }
|
||||
}
|
||||
|
||||
fun findPostById(id: Long): Optional<PostEntity> {
|
||||
return transaction { postDownloadStateRepository.findById(id) }
|
||||
fun findPostById(id: Long): PostEntity {
|
||||
return postEntityIdCache.get(id) ?: throw NoSuchElementException("Post with id = $id does not exist")
|
||||
}
|
||||
|
||||
fun findImagesByPostId(postId: Long): List<ImageEntity> {
|
||||
@@ -262,8 +287,8 @@ internal class DataTransaction(
|
||||
return transaction { imageRepository.countError() }
|
||||
}
|
||||
|
||||
fun findPostByPostId(postId: Long): Optional<PostEntity> {
|
||||
return transaction { postDownloadStateRepository.findByPostId(postId) }
|
||||
fun findPostByPostId(postId: Long): PostEntity {
|
||||
return postPostIdCache.get(postId) ?: throw NoSuchElementException("Post with postId = $postId does not exist")
|
||||
}
|
||||
|
||||
fun findThreadByThreadId(threadId: Long): Optional<ThreadEntity> {
|
||||
@@ -271,7 +296,7 @@ internal class DataTransaction(
|
||||
}
|
||||
|
||||
fun findAllNonCompletedPostIds(): List<Long> {
|
||||
return transaction { postDownloadStateRepository.findAllNonCompletedPostIds() }
|
||||
return transaction { postRepository.findAllNonCompletedPostIds() }
|
||||
}
|
||||
|
||||
fun findMetadataByPostId(postId: Long): Optional<MetadataEntity> {
|
||||
|
||||
@@ -14,7 +14,6 @@ import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager
|
||||
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder
|
||||
import org.apache.hc.core5.pool.PoolConcurrencyPolicy
|
||||
import org.apache.hc.core5.pool.PoolReusePolicy
|
||||
import org.apache.hc.core5.util.TimeValue
|
||||
import org.apache.hc.core5.util.Timeout
|
||||
|
||||
internal class HTTPService(
|
||||
@@ -41,8 +40,10 @@ internal class HTTPService(
|
||||
buildConnectionPool()
|
||||
buildClientBuilder()
|
||||
coroutineScope.launch {
|
||||
pcm.closeIdle(TimeValue.ofSeconds(60))
|
||||
delay(15000)
|
||||
while (isActive) {
|
||||
pcm.closeExpired()
|
||||
delay(60_000)
|
||||
}
|
||||
}
|
||||
coroutineScope.launch {
|
||||
eventBus
|
||||
@@ -64,8 +65,8 @@ internal class HTTPService(
|
||||
|
||||
private fun buildConnectionPool() {
|
||||
pcm = PoolingHttpClientConnectionManagerBuilder.create()
|
||||
.setPoolConcurrencyPolicy(PoolConcurrencyPolicy.STRICT)
|
||||
.setConnPoolPolicy(PoolReusePolicy.LIFO)
|
||||
.setPoolConcurrencyPolicy(PoolConcurrencyPolicy.LAX)
|
||||
.setConnPoolPolicy(PoolReusePolicy.FIFO)
|
||||
.setDefaultConnectionConfig(cc)
|
||||
.setMaxConnTotal(Int.MAX_VALUE)
|
||||
.setMaxConnPerRoute(Int.MAX_VALUE)
|
||||
@@ -83,7 +84,6 @@ internal class HTTPService(
|
||||
cc = ConnectionConfig.custom()
|
||||
.setConnectTimeout(Timeout.ofSeconds(connectionTimeout.toLong()))
|
||||
.setSocketTimeout(Timeout.ofSeconds(connectionTimeout.toLong()))
|
||||
.setTimeToLive(TimeValue.ofMinutes(10))
|
||||
.build()
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package me.vripper.services
|
||||
|
||||
import dev.failsafe.RetryPolicy
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
@@ -8,8 +9,6 @@ import kotlinx.coroutines.launch
|
||||
import me.vripper.event.EventBus
|
||||
import me.vripper.event.SettingsUpdateEvent
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import net.jodah.failsafe.RetryPolicy
|
||||
import net.jodah.failsafe.event.ExecutionAttemptedEvent
|
||||
import java.time.temporal.ChronoUnit
|
||||
|
||||
internal class RetryPolicyService(
|
||||
@@ -29,16 +28,10 @@ internal class RetryPolicyService(
|
||||
}
|
||||
}
|
||||
|
||||
fun <T> buildRetryPolicyForDownload(message: String): RetryPolicy<T> {
|
||||
return RetryPolicy<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts).onFailedAttempt {
|
||||
log.warn(message + "#${it.attemptCount} tries failed", it.lastFailure)
|
||||
}
|
||||
}
|
||||
|
||||
fun <T> buildGenericRetryPolicy(message: String): RetryPolicy<T> {
|
||||
return RetryPolicy<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts)
|
||||
.onFailedAttempt { e: ExecutionAttemptedEvent<T> ->
|
||||
log.warn(message + "#${e.attemptCount} tries failed", e.lastFailure)
|
||||
}
|
||||
fun <T> buildRetryPolicy(message: String): RetryPolicy<T> {
|
||||
return RetryPolicy.builder<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts)
|
||||
.onFailedAttempt {
|
||||
log.warn(message + "#${it.attemptCount} tries failed", it.lastException)
|
||||
}.build()
|
||||
}
|
||||
}
|
||||
@@ -2,8 +2,11 @@ package me.vripper.services
|
||||
|
||||
import com.github.benmanes.caffeine.cache.Caffeine
|
||||
import com.github.benmanes.caffeine.cache.LoadingCache
|
||||
import kotlinx.coroutines.*
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
import kotlinx.coroutines.flow.filterIsInstance
|
||||
import kotlinx.coroutines.launch
|
||||
import me.vripper.event.EventBus
|
||||
import me.vripper.event.SettingsUpdateEvent
|
||||
import me.vripper.vgapi.ThreadItem
|
||||
@@ -25,9 +28,7 @@ internal class ThreadCacheService(val eventBus: EventBus) {
|
||||
|
||||
private val cache: LoadingCache<Long, ThreadItem> =
|
||||
Caffeine.newBuilder().expireAfterWrite(20, TimeUnit.MINUTES).build { threadId ->
|
||||
runBlocking {
|
||||
ThreadLookupAPIParser(threadId).parse()
|
||||
}
|
||||
ThreadLookupAPIParser(threadId).parse()
|
||||
}
|
||||
|
||||
@Throws(ExecutionException::class)
|
||||
|
||||
@@ -12,8 +12,8 @@ import me.vripper.event.VGUserLoginEvent
|
||||
import me.vripper.exception.VripperException
|
||||
import me.vripper.model.Settings
|
||||
import me.vripper.tasks.LeaveThanksTask
|
||||
import me.vripper.utilities.GlobalScopeCoroutine
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.executorService
|
||||
import org.apache.hc.client5.http.classic.methods.HttpPost
|
||||
import org.apache.hc.client5.http.cookie.BasicCookieStore
|
||||
import org.apache.hc.client5.http.cookie.Cookie
|
||||
@@ -108,8 +108,8 @@ internal class VGAuthService(
|
||||
}
|
||||
|
||||
fun leaveThanks(postEntity: PostEntity) {
|
||||
GlobalScopeCoroutine.launch {
|
||||
LeaveThanksTask(postEntity, authenticated, context).run()
|
||||
}
|
||||
executorService.submit(
|
||||
LeaveThanksTask(postEntity, authenticated, context)
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1,14 +1,13 @@
|
||||
package me.vripper.tasks
|
||||
|
||||
import kotlinx.coroutines.launch
|
||||
import me.vripper.entities.ThreadEntity
|
||||
import me.vripper.model.Settings
|
||||
import me.vripper.model.ThreadPostId
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.SettingsService
|
||||
import me.vripper.services.ThreadCacheService
|
||||
import me.vripper.utilities.GlobalScopeCoroutine
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.executorService
|
||||
import org.koin.core.component.KoinComponent
|
||||
import org.koin.core.component.inject
|
||||
|
||||
@@ -34,15 +33,14 @@ internal class ThreadLookupTask(private val threadId: Long, private val settings
|
||||
}
|
||||
|
||||
if (threadLookupResult.postItemList.size <= settings.downloadSettings.autoQueueThreshold) {
|
||||
GlobalScopeCoroutine.launch {
|
||||
executorService.submit(
|
||||
AddPostTask(threadLookupResult.postItemList.map {
|
||||
ThreadPostId(
|
||||
it.threadId, it.postId
|
||||
)
|
||||
}).run()
|
||||
}
|
||||
})
|
||||
)
|
||||
} else {
|
||||
try {
|
||||
dataTransaction.save(
|
||||
ThreadEntity(
|
||||
title = threadLookupResult.title,
|
||||
@@ -51,9 +49,6 @@ internal class ThreadLookupTask(private val threadId: Long, private val settings
|
||||
total = threadLookupResult.postItemList.size
|
||||
)
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
package me.vripper.utilities
|
||||
|
||||
import kotlinx.coroutines.CoroutineExceptionHandler
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.SupervisorJob
|
||||
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
val errorHandler = CoroutineExceptionHandler { _, exception ->
|
||||
log.error("Unexpected error", exception)
|
||||
}
|
||||
|
||||
val GlobalScopeCoroutine = CoroutineScope(SupervisorJob() + Dispatchers.IO + errorHandler)
|
||||
@@ -0,0 +1,6 @@
|
||||
package me.vripper.utilities
|
||||
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
val executorService: ExecutorService = Executors.newVirtualThreadPerTaskExecutor()
|
||||
@@ -1,5 +1,7 @@
|
||||
package me.vripper.vgapi
|
||||
|
||||
import dev.failsafe.Failsafe
|
||||
import dev.failsafe.function.CheckedSupplier
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlinx.coroutines.sync.withPermit
|
||||
import me.vripper.exception.DownloadException
|
||||
@@ -11,8 +13,6 @@ import me.vripper.services.VGAuthService
|
||||
import me.vripper.tasks.Tasks
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.RequestLimit
|
||||
import net.jodah.failsafe.Failsafe
|
||||
import net.jodah.failsafe.function.CheckedSupplier
|
||||
import org.apache.hc.client5.http.classic.methods.HttpGet
|
||||
import org.apache.hc.core5.http.io.entity.EntityUtils
|
||||
import org.apache.hc.core5.net.URIBuilder
|
||||
@@ -42,9 +42,9 @@ internal class PostLookupAPIParser(private val threadId: Long, private val postI
|
||||
log.debug("Requesting {}", httpGet)
|
||||
Tasks.increment()
|
||||
return try {
|
||||
Failsafe.with(retryPolicyService.buildGenericRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
|
||||
Failsafe.with(retryPolicyService.buildRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
|
||||
log.error(
|
||||
"Failed to process thread $threadId, post $postId", it.failure
|
||||
"Failed to process thread $threadId, post $postId", it.exception
|
||||
)
|
||||
}.get(CheckedSupplier {
|
||||
runBlocking {
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package me.vripper.vgapi
|
||||
|
||||
import dev.failsafe.Failsafe
|
||||
import dev.failsafe.function.CheckedSupplier
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import kotlinx.coroutines.sync.withPermit
|
||||
import me.vripper.exception.DownloadException
|
||||
@@ -11,8 +13,6 @@ import me.vripper.services.VGAuthService
|
||||
import me.vripper.tasks.Tasks
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.RequestLimit
|
||||
import net.jodah.failsafe.Failsafe
|
||||
import net.jodah.failsafe.function.CheckedSupplier
|
||||
import org.apache.hc.client5.http.classic.methods.HttpGet
|
||||
import org.apache.hc.core5.http.io.entity.EntityUtils
|
||||
import org.apache.hc.core5.net.URIBuilder
|
||||
@@ -43,10 +43,10 @@ internal class ThreadLookupAPIParser(private val threadId: Long) : KoinComponent
|
||||
log.debug("Requesting {}", httpGet)
|
||||
Tasks.increment()
|
||||
return try {
|
||||
Failsafe.with(retryPolicyService.buildGenericRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
|
||||
Failsafe.with(retryPolicyService.buildRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
|
||||
log.error(
|
||||
"Failed to process thread $threadId",
|
||||
it.failure
|
||||
it.exception
|
||||
)
|
||||
}.get(CheckedSupplier {
|
||||
runBlocking {
|
||||
|
||||
@@ -23,8 +23,7 @@ class AboutFragment : Fragment("About") {
|
||||
padding = Insets(15.0, 15.0, 15.0, 15.0)
|
||||
spacing = 15.0
|
||||
imageview("icons/64x64.png")
|
||||
vbox {
|
||||
spacing = 5.0
|
||||
vbox(spacing = 5.0) {
|
||||
text("VRipper") {
|
||||
style {
|
||||
fontWeight = FontWeight.BOLD
|
||||
@@ -32,15 +31,29 @@ class AboutFragment : Fragment("About") {
|
||||
}
|
||||
}
|
||||
text("Version ${ApplicationProperties.VERSION}")
|
||||
text("Developed by death-claw and VRipper working group")
|
||||
hyperlink("Home Page") {
|
||||
action {
|
||||
openLink("https://github.com/death-claw/vripper-project")
|
||||
text("Developed by dev-claw and VRipper working group")
|
||||
hbox(spacing = 5.0) {
|
||||
hyperlink {
|
||||
imageview("icons/github-mark.png").apply {
|
||||
isPreserveRatio = true
|
||||
fitHeight = 32.0
|
||||
}
|
||||
action {
|
||||
openLink("https://github.com/dev-claw/vripper-project")
|
||||
}
|
||||
}
|
||||
hyperlink {
|
||||
imageview("icons/buymeacoffee-logo.png").apply {
|
||||
isPreserveRatio = true
|
||||
fitHeight = 32.0
|
||||
}
|
||||
action {
|
||||
openLink("https://buymeacoffee.com/devclaw")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
tab("System") {
|
||||
form {
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 6.2 KiB |
Reference in New Issue
Block a user