Compare commits

...
Author SHA1 Message Date
dev-claw 804c3d0c04 fixes #208 2024-12-27 19:11:43 +01:00
23 changed files with 206 additions and 204 deletions
+2 -2
View File
@@ -25,7 +25,7 @@
<koin.version>4.0.0</koin.version>
<h2.version>2.2.224</h2.version>
<htmlcleaner.version>2.29</htmlcleaner.version>
<failsafe.version>2.4.4</failsafe.version>
<failsafe.version>3.3.2</failsafe.version>
<caffeine.version>3.1.8</caffeine.version>
<jna-platform.version>5.14.0</jna-platform.version>
<kotlinx-serialization-json.version>1.7.3</kotlinx-serialization-json.version>
@@ -142,8 +142,8 @@
<version>${htmlcleaner.version}</version>
</dependency>
<dependency>
<groupId>dev.failsafe</groupId>
<artifactId>failsafe</artifactId>
<groupId>net.jodah</groupId>
<version>${failsafe.version}</version>
</dependency>
<dependency>
+1 -1
View File
@@ -69,7 +69,7 @@
</dependency>
<dependency>
<artifactId>failsafe</artifactId>
<groupId>net.jodah</groupId>
<groupId>dev.failsafe</groupId>
</dependency>
<dependency>
<artifactId>caffeine</artifactId>
@@ -2,11 +2,11 @@ package me.vripper
import me.vripper.data.repositories.ImageRepository
import me.vripper.data.repositories.MetadataRepository
import me.vripper.data.repositories.PostDownloadStateRepository
import me.vripper.data.repositories.PostRepository
import me.vripper.data.repositories.ThreadRepository
import me.vripper.data.repositories.impl.ImageRepositoryImpl
import me.vripper.data.repositories.impl.MetadataRepositoryImpl
import me.vripper.data.repositories.impl.PostDownloadStateRepositoryImpl
import me.vripper.data.repositories.impl.PostRepositoryImpl
import me.vripper.data.repositories.impl.ThreadRepositoryImpl
import me.vripper.download.DownloadService
import me.vripper.event.EventBus
@@ -29,8 +29,8 @@ val coreModule = module {
single<ImageRepository> {
ImageRepositoryImpl()
}
single<PostDownloadStateRepository> {
PostDownloadStateRepositoryImpl()
single<PostRepository> {
PostRepositoryImpl()
}
single<MetadataRepository> {
MetadataRepositoryImpl()
@@ -1,12 +1,11 @@
package me.vripper.data.repositories
import me.vripper.entities.PostEntity
import java.util.*
internal interface PostDownloadStateRepository {
internal interface PostRepository {
fun save(postEntities: List<PostEntity>): List<PostEntity>
fun findByPostId(postId: Long): Optional<PostEntity>
fun findById(id: Long): Optional<PostEntity>
fun findByPostId(postId: Long): PostEntity?
fun findById(id: Long): PostEntity?
fun findCompleted(): List<Long>
fun findAll(): List<PostEntity>
fun existByPostId(postId: Long): Boolean
@@ -1,6 +1,6 @@
package me.vripper.data.repositories.impl
import me.vripper.data.repositories.PostDownloadStateRepository
import me.vripper.data.repositories.PostRepository
import me.vripper.data.tables.PostTable
import me.vripper.entities.PostEntity
import me.vripper.entities.Status
@@ -8,10 +8,9 @@ import org.jetbrains.exposed.sql.*
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.transactions.TransactionManager
import java.sql.Connection
import java.util.*
internal class PostDownloadStateRepositoryImpl :
PostDownloadStateRepository {
internal class PostRepositoryImpl :
PostRepository {
private val delimiter = ";"
@@ -37,15 +36,11 @@ internal class PostDownloadStateRepositoryImpl :
}.map(::transform)
}
override fun findByPostId(postId: Long): Optional<PostEntity> {
override fun findByPostId(postId: Long): PostEntity? {
val result = PostTable.selectAll().where {
PostTable.postId eq postId
}.map(::transform)
return if (result.isEmpty()) {
Optional.empty()
} else {
Optional.of(result.first())
}
return result.firstOrNull()
}
override fun findCompleted(): List<Long> {
@@ -54,16 +49,11 @@ internal class PostDownloadStateRepositoryImpl :
}.map { it[PostTable.postId] }
}
override fun findById(id: Long): Optional<PostEntity> {
override fun findById(id: Long): PostEntity? {
val result = PostTable.selectAll().where {
PostTable.id eq id
}.map { transform(it) }
return if (result.isEmpty()) {
Optional.empty()
} else {
Optional.of(result.first())
}
return result.firstOrNull()
}
override fun findAll(): List<PostEntity> {
@@ -1,7 +1,8 @@
package me.vripper.download
import dev.failsafe.Failsafe
import dev.failsafe.RetryPolicy
import kotlinx.coroutines.Runnable
import kotlinx.coroutines.launch
import me.vripper.entities.ImageEntity
import me.vripper.entities.PostEntity
import me.vripper.entities.Status
@@ -16,10 +17,8 @@ import me.vripper.services.DataTransaction
import me.vripper.services.RetryPolicyService
import me.vripper.services.SettingsService
import me.vripper.services.VGAuthService
import me.vripper.utilities.GlobalScopeCoroutine
import me.vripper.utilities.LoggerDelegate
import net.jodah.failsafe.Failsafe
import net.jodah.failsafe.RetryPolicy
import me.vripper.utilities.executorService
import org.jetbrains.exposed.sql.transactions.transaction
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
@@ -46,14 +45,14 @@ internal class DownloadService(
lock.withLock {
candidates.addAll(getCandidates(candidateCount()))
candidates.forEach {
if (canRun(it.context.imageEntity.host)) {
if (canRun(it.imageEntity.host)) {
accepted.add(it)
running[it.context.imageEntity.host]!!.add(it)
log.debug("${it.context.imageEntity.url} accepted to run")
running[it.imageEntity.host]!!.add(it)
log.debug("${it.imageEntity.url} accepted to run")
}
}
accepted.forEach {
pending[it.context.imageEntity.host]?.remove(it)
pending[it.imageEntity.host]?.remove(it)
scheduleForDownload(it)
}
accepted.clear()
@@ -144,18 +143,18 @@ internal class DownloadService(
}
private fun isPending(postId: Long): Boolean {
return pending.values.flatten().any { it.context.imageEntity.postId == postId }
return pending.values.flatten().any { it.imageEntity.postId == postId }
}
private fun isRunning(postId: Long): Boolean {
return running.values.flatten().any { it.context.imageEntity.postId == postId }
return running.values.flatten().any { it.imageEntity.postId == postId }
}
private fun stopAll() {
lock.withLock {
pending.values.clear()
running.values.flatten().forEach { obj: ImageDownloadRunnable -> obj.stop() }
while (running.values.flatten().count { !it.context.completed } > 0) {
while (running.values.flatten().count { !it.completed } > 0) {
Thread.sleep(100)
}
dataTransaction.findAllNonCompletedPostIds().forEach {
@@ -169,13 +168,13 @@ internal class DownloadService(
lock.withLock {
for (postId in postIds) {
pending.values.forEach { pending ->
pending.removeIf { it.context.imageEntity.postId == postId }
pending.removeIf { it.imageEntity.postId == postId }
}
running.values.flatten()
.filter { p: ImageDownloadRunnable -> p.context.imageEntity.postId == postId }
.filter { p: ImageDownloadRunnable -> p.imageEntity.postId == postId }
.forEach { obj: ImageDownloadRunnable -> obj.stop() }
while (running.values.flatten()
.count { !it.context.completed && it.context.imageEntity.postId == postId } > 0
.count { !it.completed && it.imageEntity.postId == postId } > 0
) {
Thread.sleep(100)
}
@@ -213,7 +212,7 @@ internal class DownloadService(
val list: List<ImageDownloadRunnable> =
pending[host]!!.sortedWith(Comparator.comparingInt<ImageDownloadRunnable> { it.postRank }
.thenComparingInt { it.context.imageEntity.index })
.thenComparingInt { it.imageEntity.index })
for (imageDownloadRunnable in list) {
val count = hostIntegerMap[host] ?: 0
@@ -229,47 +228,42 @@ internal class DownloadService(
}
private fun scheduleForDownload(imageDownloadRunnable: ImageDownloadRunnable) {
log.debug("Scheduling a job for ${imageDownloadRunnable.context.imageEntity.url}")
GlobalScopeCoroutine.launch {
eventBus.publishEvent(QueueStateEvent(QueueState(runningCount(), pendingCount())))
try {
Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicyForDownload("Failed to download ${imageDownloadRunnable.context.imageEntity.url}: "))
.onFailure {
log.error(
"Failed to download ${imageDownloadRunnable.context.imageEntity.url} after ${it.attemptCount} tries",
it.failure
)
val image = imageDownloadRunnable.context.imageEntity
image.status = Status.ERROR
dataTransaction.updateImage(image)
}
.onComplete {
afterJobFinish(imageDownloadRunnable)
eventBus.publishEvent(
QueueStateEvent(
QueueState(
runningCount(), pendingCount()
)
)
)
eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataTransaction.countImagesInError())))
log.debug(
"Finished downloading ${imageDownloadRunnable.context.imageEntity.url}"
)
}.run(imageDownloadRunnable::run)
} catch (e: Exception) {
log.error("Download Failure", e)
log.debug("Scheduling a job for ${imageDownloadRunnable.imageEntity.url}")
eventBus.publishEvent(QueueStateEvent(QueueState(runningCount(), pendingCount())))
Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicy("Failed to download ${imageDownloadRunnable.imageEntity.url}: "))
.with(executorService)
.onFailure {
log.error(
"Failed to download ${imageDownloadRunnable.imageEntity.url} after ${it.attemptCount} tries",
it.exception
)
val image = imageDownloadRunnable.imageEntity
image.status = Status.ERROR
dataTransaction.updateImage(image)
}
}
.onComplete {
afterJobFinish(imageDownloadRunnable)
eventBus.publishEvent(
QueueStateEvent(
QueueState(
runningCount(), pendingCount()
)
)
)
eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataTransaction.countImagesInError())))
log.debug(
"Finished downloading ${imageDownloadRunnable.imageEntity.url}"
)
}.runAsync(imageDownloadRunnable)
}
private fun afterJobFinish(imageDownloadRunnable: ImageDownloadRunnable) {
lock.withLock {
val image = imageDownloadRunnable.context.imageEntity
val image = imageDownloadRunnable.imageEntity
running[image.host]!!.remove(imageDownloadRunnable)
if (!isPending(image.postId) && !isRunning(
image.postId
) && !imageDownloadRunnable.context.stopped
) && !imageDownloadRunnable.stopped
) {
dataTransaction.finishPost(image.postId, true)
}
@@ -15,8 +15,6 @@ internal class ImageDownloadContext(val imageEntity: ImageEntity, val settings:
HttpClientContext.create().apply { cookieStore = BasicCookieStore() }
val requests = mutableListOf<HttpUriRequestBase>()
val postId = imageEntity.postIdRef
var stopped = false
var completed = false
fun cancelCoroutines() {
runBlocking {
@@ -1,5 +1,6 @@
package me.vripper.download
import dev.failsafe.function.CheckedRunnable
import me.vripper.entities.ImageEntity
import me.vripper.entities.Status
import me.vripper.exception.DownloadException
@@ -13,7 +14,6 @@ import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.PathUtils.getExtension
import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
import me.vripper.utilities.PathUtils.sanitize
import net.jodah.failsafe.function.CheckedRunnable
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.io.IOException
@@ -26,23 +26,23 @@ import kotlin.io.path.Path
import kotlin.io.path.pathString
internal class ImageDownloadRunnable(
private val imageEntity: ImageEntity, val postRank: Int, private val settings: Settings
val imageEntity: ImageEntity, val postRank: Int, private val settings: Settings
) : KoinComponent, CheckedRunnable {
private val log by LoggerDelegate()
private val dataTransaction: DataTransaction by inject()
private val hosts: List<Host> = getKoin().getAll()
var completed = false
var stopped = false
val context: ImageDownloadContext = ImageDownloadContext(imageEntity, settings)
private lateinit var context: ImageDownloadContext
@Throws(DownloadException::class)
fun download() {
try {
imageEntity.status = Status.DOWNLOADING
imageEntity.downloaded = 0
dataTransaction.updateImage(imageEntity)
synchronized(imageEntity.postId.toString().intern()) {
val post = dataTransaction.findPostById(context.postId).orElseThrow()
val post = dataTransaction.findPostById(context.postId)
if (post.status != Status.DOWNLOADING) {
post.status = Status.DOWNLOADING
dataTransaction.updatePost(post)
@@ -54,7 +54,7 @@ internal class ImageDownloadRunnable(
log.debug("Resolved name for ${imageEntity.url}: ${downloadedImage.name}")
log.debug("Downloaded image {} to {}", imageEntity.url, downloadedImage.path)
synchronized(imageEntity.postId.toString().intern()) {
val post = dataTransaction.findPostById(context.postId).orElseThrow()
val post = dataTransaction.findPostById(context.postId)
val downloadDirectory = Path(post.downloadDirectory, post.folderName).pathString
checkImageTypeAndRename(
downloadDirectory, downloadedImage, imageEntity.index
@@ -70,7 +70,7 @@ internal class ImageDownloadRunnable(
dataTransaction.updateImage(imageEntity)
}
} catch (e: Exception) {
if (context.stopped) {
if (stopped) {
return
}
imageEntity.status = Status.ERROR
@@ -119,19 +119,26 @@ internal class ImageDownloadRunnable(
}
}
@Throws(Exception::class)
override fun run() {
context = ImageDownloadContext(imageEntity, settings)
try {
if (context.stopped) {
if (stopped) {
return
}
download()
} finally {
context.completed = true
completed = true
context.cancelCoroutines()
}
}
fun stop() {
stopped = true
context.requests.forEach { it.abort() }
context.cancelCoroutines()
dataTransaction.updateImage(context.imageEntity)
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other == null || javaClass != other.javaClass) return false
@@ -142,11 +149,4 @@ internal class ImageDownloadRunnable(
override fun hashCode(): Int {
return Objects.hash(imageEntity.id)
}
fun stop() {
context.requests.forEach { it.abort() }
context.cancelCoroutines()
context.stopped = true
dataTransaction.updateImage(context.imageEntity)
}
}
@@ -106,7 +106,7 @@ internal abstract class Host(
return BufferedOutputStream(Files.newOutputStream(tempImage)).use { bos ->
val image = context.imageEntity
synchronized(image.postId.toString().intern()) {
val post = dataTransaction.findPostById(context.postId).orElseThrow()
val post = dataTransaction.findPostById(context.postId)
val size = if (image.size < 0) {
response.entity.contentLength
} else {
@@ -134,7 +134,7 @@ internal abstract class Host(
}
}
while (response.entity.content.read(buffer)
.also { read = it } != -1 && !context.stopped
.also { read = it } != -1
) {
bos.write(buffer, 0, read)
image.downloaded += read
@@ -1,7 +1,6 @@
package me.vripper.services
import kotlinx.coroutines.flow.*
import kotlinx.coroutines.launch
import kotlinx.coroutines.time.sample
import kotlinx.serialization.json.Json
import me.vripper.download.DownloadService
@@ -13,9 +12,9 @@ import me.vripper.tasks.AddPostTask
import me.vripper.tasks.ThreadLookupTask
import me.vripper.utilities.ApplicationProperties
import me.vripper.utilities.ApplicationProperties.VRIPPER_DIR
import me.vripper.utilities.GlobalScopeCoroutine
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.PathUtils
import me.vripper.utilities.executorService
import org.h2.jdbc.JdbcSQLNonTransientConnectionException
import java.sql.DriverManager
import java.time.Duration
@@ -59,17 +58,17 @@ internal class AppEndpointService(
threadId = m.group(1).toLong()
postId = m.group(4)?.toLong()
if (postId == null) {
GlobalScopeCoroutine.launch {
executorService.submit(
ThreadLookupTask(
threadId, settingsService.settings
).run()
}
)
)
} else {
GlobalScopeCoroutine.launch {
executorService.submit(
AddPostTask(
listOf(ThreadPostId(threadId, postId))
).run()
}
)
)
}
} else {
log.error("Invalid link $link, link is missing the threadId")
@@ -81,15 +80,13 @@ internal class AppEndpointService(
override suspend fun restartAll(posIds: List<Long>) {
lock.withLock {
downloadService.restartAll(posIds.map { dataTransaction.findPostByPostId(it) }.filter { it.isPresent }
.map { it.get() })
downloadService.restartAll(posIds.filter { dataTransaction.exists(it) }
.map { dataTransaction.findPostByPostId(it) })
}
}
override suspend fun download(posts: List<ThreadPostId>) {
GlobalScopeCoroutine.launch {
AddPostTask(posts).run()
}
executorService.submit(AddPostTask(posts))
}
override suspend fun stopAll(postIdList: List<Long>) {
@@ -185,16 +182,21 @@ internal class AppEndpointService(
}
override suspend fun rename(postId: Long, newName: String) {
GlobalScopeCoroutine.launch {
executorService.submit {
synchronized(postId.toString().intern()) {
dataTransaction.findPostByPostId(postId).ifPresent { post ->
if (Path(post.downloadDirectory, post.folderName).exists()) {
PathUtils.rename(
dataTransaction.findImagesByPostId(postId), post.downloadDirectory, post.folderName, newName
)
if (dataTransaction.exists(postId)) {
dataTransaction.findPostByPostId(postId).let { post ->
if (Path(post.downloadDirectory, post.folderName).exists()) {
PathUtils.rename(
dataTransaction.findImagesByPostId(postId),
post.downloadDirectory,
post.folderName,
newName
)
}
post.folderName = PathUtils.sanitize(newName)
dataTransaction.updatePost(post)
}
post.folderName = PathUtils.sanitize(newName)
dataTransaction.updatePost(post)
}
}
}
@@ -251,7 +253,7 @@ internal class AppEndpointService(
}
override suspend fun findPost(postId: Long): Post {
return mapper(dataTransaction.findPostByPostId(postId).orElseThrow())
return mapper(dataTransaction.findPostByPostId(postId))
}
override suspend fun findImagesByPostId(postId: Long): List<Image> {
@@ -353,7 +355,7 @@ internal class AppEndpointService(
val addedAt = it.getTimestamp("ADDED_AT")
val folderName = it.getString("FOLDER_NAME") ?: ""
val exists = dataTransaction.findPostByPostId(postId).isPresent
val exists = dataTransaction.exists(postId)
if (exists) {
continue
}
@@ -1,8 +1,10 @@
package me.vripper.services
import com.github.benmanes.caffeine.cache.Caffeine
import com.github.benmanes.caffeine.cache.LoadingCache
import me.vripper.data.repositories.ImageRepository
import me.vripper.data.repositories.MetadataRepository
import me.vripper.data.repositories.PostDownloadStateRepository
import me.vripper.data.repositories.PostRepository
import me.vripper.data.repositories.ThreadRepository
import me.vripper.entities.*
import me.vripper.event.*
@@ -12,12 +14,13 @@ import me.vripper.utilities.PathUtils.sanitize
import me.vripper.vgapi.PostItem
import org.jetbrains.exposed.sql.transactions.transaction
import java.util.*
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicInteger
import kotlin.io.path.pathString
internal class DataTransaction(
private val settingsService: SettingsService,
private val postDownloadStateRepository: PostDownloadStateRepository,
private val postRepository: PostRepository,
private val imageRepository: ImageRepository,
private val threadRepository: ThreadRepository,
private val metadataRepository: MetadataRepository,
@@ -25,15 +28,24 @@ internal class DataTransaction(
) {
private val nextRank = AtomicInteger(transaction { getQueuePosition() }?.plus(1) ?: 0)
private val postEntityIdCache: LoadingCache<Long, PostEntity> =
Caffeine.newBuilder().expireAfterAccess(5, TimeUnit.MINUTES).build { id ->
transaction { postRepository.findById(id) }
}
private val postPostIdCache: LoadingCache<Long, PostEntity> =
Caffeine.newBuilder().expireAfterAccess(5, TimeUnit.MINUTES).build { id ->
transaction { postRepository.findByPostId(id) }
}
private fun save(postEntities: List<PostEntity>): List<PostEntity> {
return transaction { postDownloadStateRepository.save(postEntities) }
return transaction { postRepository.save(postEntities) }
}
fun saveAndNotify(postEntity: PostEntity, images: List<ImageEntity>) {
val savedPost = transaction {
val savedPost =
postDownloadStateRepository.save(listOf(postEntity.copy(rank = nextRank.andIncrement))).first()
postRepository.save(listOf(postEntity.copy(rank = nextRank.andIncrement))).first()
save(images.map { it.copy(postIdRef = savedPost.id) })
savedPost
}
@@ -41,12 +53,18 @@ internal class DataTransaction(
}
fun updatePosts(postEntities: List<PostEntity>) {
transaction { postDownloadStateRepository.update(postEntities) }
transaction { postRepository.update(postEntities) }
postEntities.forEach { postEntity ->
postPostIdCache.put(postEntity.postId, postEntity)
postEntityIdCache.put(postEntity.id, postEntity)
}
eventBus.publishEvent(PostUpdateEvent(postEntities))
}
fun updatePost(postEntity: PostEntity) {
transaction { postDownloadStateRepository.update(postEntity) }
transaction { postRepository.update(postEntity) }
postPostIdCache.put(postEntity.postId, postEntity)
postEntityIdCache.put(postEntity.id, postEntity)
eventBus.publishEvent(PostUpdateEvent(listOf(postEntity)))
}
@@ -75,7 +93,10 @@ internal class DataTransaction(
}
fun exists(postId: Long): Boolean {
return transaction { postDownloadStateRepository.existByPostId(postId) }
if (postPostIdCache.getIfPresent(postId) != null) {
return true
}
return transaction { postRepository.existByPostId(postId) }
}
@Synchronized
@@ -128,7 +149,7 @@ internal class DataTransaction(
}
private fun getQueuePosition(): Int? {
return postDownloadStateRepository.findMaxRank()
return postRepository.findMaxRank()
}
private fun save(imageEntities: List<ImageEntity>) {
@@ -136,7 +157,7 @@ internal class DataTransaction(
}
fun finishPost(postId: Long, automatic: Boolean = false) {
val post = findPostByPostId(postId).orElseThrow()
val post = findPostByPostId(postId)
val imagesInErrorStatus = findByPostIdAndIsError(post.postId)
if (imagesInErrorStatus.isNotEmpty()) {
post.status = Status.ERROR
@@ -167,9 +188,13 @@ internal class DataTransaction(
transaction {
metadataRepository.deleteAllByPostId(postIds)
imageRepository.deleteAllByPostId(postIds)
postDownloadStateRepository.deleteAll(postIds)
postRepository.deleteAll(postIds)
sortPostsByRank()
}
postIds.forEach { postId ->
postPostIdCache.get(postId)?.let { postEntityIdCache.invalidate(it.id) }
postPostIdCache.invalidate(postId)
}
eventBus.publishEvent(PostDeleteEvent(postIds = postIds))
eventBus.publishEvent(ErrorCountEvent(ErrorCount(countImagesInError())))
}
@@ -180,7 +205,7 @@ internal class DataTransaction(
}
fun clearCompleted(): List<Long> {
val completed = transaction { postDownloadStateRepository.findCompleted() }
val completed = transaction { postRepository.findCompleted() }
remove(completed)
return completed
}
@@ -225,15 +250,15 @@ internal class DataTransaction(
}
fun setDownloadingToStopped() {
transaction { postDownloadStateRepository.setDownloadingToStopped() }
transaction { postRepository.setDownloadingToStopped() }
}
fun findAllPosts(): List<PostEntity> {
return transaction { postDownloadStateRepository.findAll() }
return transaction { postRepository.findAll() }
}
fun findPostById(id: Long): Optional<PostEntity> {
return transaction { postDownloadStateRepository.findById(id) }
fun findPostById(id: Long): PostEntity {
return postEntityIdCache.get(id) ?: throw NoSuchElementException("Post with id = $id does not exist")
}
fun findImagesByPostId(postId: Long): List<ImageEntity> {
@@ -262,8 +287,8 @@ internal class DataTransaction(
return transaction { imageRepository.countError() }
}
fun findPostByPostId(postId: Long): Optional<PostEntity> {
return transaction { postDownloadStateRepository.findByPostId(postId) }
fun findPostByPostId(postId: Long): PostEntity {
return postPostIdCache.get(postId) ?: throw NoSuchElementException("Post with postId = $postId does not exist")
}
fun findThreadByThreadId(threadId: Long): Optional<ThreadEntity> {
@@ -271,7 +296,7 @@ internal class DataTransaction(
}
fun findAllNonCompletedPostIds(): List<Long> {
return transaction { postDownloadStateRepository.findAllNonCompletedPostIds() }
return transaction { postRepository.findAllNonCompletedPostIds() }
}
fun findMetadataByPostId(postId: Long): Optional<MetadataEntity> {
@@ -14,7 +14,6 @@ import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder
import org.apache.hc.core5.pool.PoolConcurrencyPolicy
import org.apache.hc.core5.pool.PoolReusePolicy
import org.apache.hc.core5.util.TimeValue
import org.apache.hc.core5.util.Timeout
internal class HTTPService(
@@ -41,8 +40,10 @@ internal class HTTPService(
buildConnectionPool()
buildClientBuilder()
coroutineScope.launch {
pcm.closeIdle(TimeValue.ofSeconds(60))
delay(15000)
while (isActive) {
pcm.closeExpired()
delay(60_000)
}
}
coroutineScope.launch {
eventBus
@@ -64,8 +65,8 @@ internal class HTTPService(
private fun buildConnectionPool() {
pcm = PoolingHttpClientConnectionManagerBuilder.create()
.setPoolConcurrencyPolicy(PoolConcurrencyPolicy.STRICT)
.setConnPoolPolicy(PoolReusePolicy.LIFO)
.setPoolConcurrencyPolicy(PoolConcurrencyPolicy.LAX)
.setConnPoolPolicy(PoolReusePolicy.FIFO)
.setDefaultConnectionConfig(cc)
.setMaxConnTotal(Int.MAX_VALUE)
.setMaxConnPerRoute(Int.MAX_VALUE)
@@ -83,7 +84,6 @@ internal class HTTPService(
cc = ConnectionConfig.custom()
.setConnectTimeout(Timeout.ofSeconds(connectionTimeout.toLong()))
.setSocketTimeout(Timeout.ofSeconds(connectionTimeout.toLong()))
.setTimeToLive(TimeValue.ofMinutes(10))
.build()
}
@@ -1,5 +1,6 @@
package me.vripper.services
import dev.failsafe.RetryPolicy
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
@@ -8,8 +9,6 @@ import kotlinx.coroutines.launch
import me.vripper.event.EventBus
import me.vripper.event.SettingsUpdateEvent
import me.vripper.utilities.LoggerDelegate
import net.jodah.failsafe.RetryPolicy
import net.jodah.failsafe.event.ExecutionAttemptedEvent
import java.time.temporal.ChronoUnit
internal class RetryPolicyService(
@@ -29,16 +28,10 @@ internal class RetryPolicyService(
}
}
fun <T> buildRetryPolicyForDownload(message: String): RetryPolicy<T> {
return RetryPolicy<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts).onFailedAttempt {
log.warn(message + "#${it.attemptCount} tries failed", it.lastFailure)
}
}
fun <T> buildGenericRetryPolicy(message: String): RetryPolicy<T> {
return RetryPolicy<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts)
.onFailedAttempt { e: ExecutionAttemptedEvent<T> ->
log.warn(message + "#${e.attemptCount} tries failed", e.lastFailure)
}
fun <T> buildRetryPolicy(message: String): RetryPolicy<T> {
return RetryPolicy.builder<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts)
.onFailedAttempt {
log.warn(message + "#${it.attemptCount} tries failed", it.lastException)
}.build()
}
}
@@ -2,8 +2,11 @@ package me.vripper.services
import com.github.benmanes.caffeine.cache.Caffeine
import com.github.benmanes.caffeine.cache.LoadingCache
import kotlinx.coroutines.*
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.flow.filterIsInstance
import kotlinx.coroutines.launch
import me.vripper.event.EventBus
import me.vripper.event.SettingsUpdateEvent
import me.vripper.vgapi.ThreadItem
@@ -25,9 +28,7 @@ internal class ThreadCacheService(val eventBus: EventBus) {
private val cache: LoadingCache<Long, ThreadItem> =
Caffeine.newBuilder().expireAfterWrite(20, TimeUnit.MINUTES).build { threadId ->
runBlocking {
ThreadLookupAPIParser(threadId).parse()
}
ThreadLookupAPIParser(threadId).parse()
}
@Throws(ExecutionException::class)
@@ -12,8 +12,8 @@ import me.vripper.event.VGUserLoginEvent
import me.vripper.exception.VripperException
import me.vripper.model.Settings
import me.vripper.tasks.LeaveThanksTask
import me.vripper.utilities.GlobalScopeCoroutine
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.executorService
import org.apache.hc.client5.http.classic.methods.HttpPost
import org.apache.hc.client5.http.cookie.BasicCookieStore
import org.apache.hc.client5.http.cookie.Cookie
@@ -108,8 +108,8 @@ internal class VGAuthService(
}
fun leaveThanks(postEntity: PostEntity) {
GlobalScopeCoroutine.launch {
LeaveThanksTask(postEntity, authenticated, context).run()
}
executorService.submit(
LeaveThanksTask(postEntity, authenticated, context)
)
}
}
@@ -1,14 +1,13 @@
package me.vripper.tasks
import kotlinx.coroutines.launch
import me.vripper.entities.ThreadEntity
import me.vripper.model.Settings
import me.vripper.model.ThreadPostId
import me.vripper.services.DataTransaction
import me.vripper.services.SettingsService
import me.vripper.services.ThreadCacheService
import me.vripper.utilities.GlobalScopeCoroutine
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.executorService
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
@@ -34,15 +33,14 @@ internal class ThreadLookupTask(private val threadId: Long, private val settings
}
if (threadLookupResult.postItemList.size <= settings.downloadSettings.autoQueueThreshold) {
GlobalScopeCoroutine.launch {
executorService.submit(
AddPostTask(threadLookupResult.postItemList.map {
ThreadPostId(
it.threadId, it.postId
)
}).run()
}
})
)
} else {
try {
dataTransaction.save(
ThreadEntity(
title = threadLookupResult.title,
@@ -51,9 +49,6 @@ internal class ThreadLookupTask(private val threadId: Long, private val settings
total = threadLookupResult.postItemList.size
)
)
} catch (e: Exception) {
e.printStackTrace()
}
}
}
} catch (e: Exception) {
@@ -1,14 +0,0 @@
package me.vripper.utilities
import kotlinx.coroutines.CoroutineExceptionHandler
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
private val log by LoggerDelegate()
val errorHandler = CoroutineExceptionHandler { _, exception ->
log.error("Unexpected error", exception)
}
val GlobalScopeCoroutine = CoroutineScope(SupervisorJob() + Dispatchers.IO + errorHandler)
@@ -0,0 +1,6 @@
package me.vripper.utilities
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
val executorService: ExecutorService = Executors.newVirtualThreadPerTaskExecutor()
@@ -1,5 +1,7 @@
package me.vripper.vgapi
import dev.failsafe.Failsafe
import dev.failsafe.function.CheckedSupplier
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.withPermit
import me.vripper.exception.DownloadException
@@ -11,8 +13,6 @@ import me.vripper.services.VGAuthService
import me.vripper.tasks.Tasks
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.RequestLimit
import net.jodah.failsafe.Failsafe
import net.jodah.failsafe.function.CheckedSupplier
import org.apache.hc.client5.http.classic.methods.HttpGet
import org.apache.hc.core5.http.io.entity.EntityUtils
import org.apache.hc.core5.net.URIBuilder
@@ -42,9 +42,9 @@ internal class PostLookupAPIParser(private val threadId: Long, private val postI
log.debug("Requesting {}", httpGet)
Tasks.increment()
return try {
Failsafe.with(retryPolicyService.buildGenericRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
Failsafe.with(retryPolicyService.buildRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
log.error(
"Failed to process thread $threadId, post $postId", it.failure
"Failed to process thread $threadId, post $postId", it.exception
)
}.get(CheckedSupplier {
runBlocking {
@@ -1,5 +1,7 @@
package me.vripper.vgapi
import dev.failsafe.Failsafe
import dev.failsafe.function.CheckedSupplier
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.withPermit
import me.vripper.exception.DownloadException
@@ -11,8 +13,6 @@ import me.vripper.services.VGAuthService
import me.vripper.tasks.Tasks
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.RequestLimit
import net.jodah.failsafe.Failsafe
import net.jodah.failsafe.function.CheckedSupplier
import org.apache.hc.client5.http.classic.methods.HttpGet
import org.apache.hc.core5.http.io.entity.EntityUtils
import org.apache.hc.core5.net.URIBuilder
@@ -43,10 +43,10 @@ internal class ThreadLookupAPIParser(private val threadId: Long) : KoinComponent
log.debug("Requesting {}", httpGet)
Tasks.increment()
return try {
Failsafe.with(retryPolicyService.buildGenericRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
Failsafe.with(retryPolicyService.buildRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
log.error(
"Failed to process thread $threadId",
it.failure
it.exception
)
}.get(CheckedSupplier {
runBlocking {
@@ -23,8 +23,7 @@ class AboutFragment : Fragment("About") {
padding = Insets(15.0, 15.0, 15.0, 15.0)
spacing = 15.0
imageview("icons/64x64.png")
vbox {
spacing = 5.0
vbox(spacing = 5.0) {
text("VRipper") {
style {
fontWeight = FontWeight.BOLD
@@ -32,15 +31,29 @@ class AboutFragment : Fragment("About") {
}
}
text("Version ${ApplicationProperties.VERSION}")
text("Developed by death-claw and VRipper working group")
hyperlink("Home Page") {
action {
openLink("https://github.com/death-claw/vripper-project")
text("Developed by dev-claw and VRipper working group")
hbox(spacing = 5.0) {
hyperlink {
imageview("icons/github-mark.png").apply {
isPreserveRatio = true
fitHeight = 32.0
}
action {
openLink("https://github.com/dev-claw/vripper-project")
}
}
hyperlink {
imageview("icons/buymeacoffee-logo.png").apply {
isPreserveRatio = true
fitHeight = 32.0
}
action {
openLink("https://buymeacoffee.com/devclaw")
}
}
}
}
}
}
tab("System") {
form {
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.2 KiB