diff --git a/pom.xml b/pom.xml index ff3d0b2..8f7d1af 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ ${revision} 21 - 1.9.20 + 1.9.22 false ${maven.build.timestamp} yyyy-MM-dd HH:mm:ss @@ -150,6 +150,11 @@ org.jetbrains.kotlin ${kotlin.version} + + org.jetbrains.kotlinx + kotlinx-serialization-json + 1.6.2 + jcl-over-slf4j org.slf4j diff --git a/vripper-core/pom.xml b/vripper-core/pom.xml index 4652341..c32cb98 100644 --- a/vripper-core/pom.xml +++ b/vripper-core/pom.xml @@ -43,10 +43,6 @@ io.insert-koin koin-core-jvm - - jackson-module-kotlin - com.fasterxml.jackson.module - h2 com.h2database @@ -71,18 +67,6 @@ caffeine com.github.ben-manes.caffeine - - jackson-databind - com.fasterxml.jackson.core - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 - commons-codec commons-codec @@ -100,10 +84,14 @@ com.mattbertolini runtime - - io.projectreactor - reactor-core - + + + + + + org.jetbrains.kotlinx + kotlinx-serialization-json + @@ -121,14 +109,21 @@ compile - - - src/main/kotlin - target/generated-sources/annotations - - + + 1.8 + + kotlinx-serialization + + + + + org.jetbrains.kotlin + kotlin-maven-serialization + ${kotlin.version} + + maven-jar-plugin diff --git a/vripper-core/src/main/kotlin/me/vripper/Module.kt b/vripper-core/src/main/kotlin/me/vripper/Module.kt index 496b8b4..e447877 100644 --- a/vripper-core/src/main/kotlin/me/vripper/Module.kt +++ b/vripper-core/src/main/kotlin/me/vripper/Module.kt @@ -55,6 +55,9 @@ val coreModule = module { single { AppEndpointService(get(), get(), get(), get()) } + single { + MetadataService(get(), get(), get(), get()) + } single { AcidimgHost(get(), get(), get()) } bind Host::class diff --git a/vripper-core/src/main/kotlin/me/vripper/download/DownloadService.kt b/vripper-core/src/main/kotlin/me/vripper/download/DownloadService.kt index a303dcc..aca789c 100644 --- a/vripper-core/src/main/kotlin/me/vripper/download/DownloadService.kt +++ b/vripper-core/src/main/kotlin/me/vripper/download/DownloadService.kt @@ -1,6 +1,6 @@ package me.vripper.download -import kotlinx.coroutines.Runnable +import kotlinx.coroutines.* import me.vripper.entities.Image import me.vripper.entities.LogEntry import me.vripper.entities.Post @@ -37,6 +37,7 @@ class DownloadService( private val pending: MutableMap> = mutableMapOf() private val lock = ReentrantLock() private val condition = lock.newCondition() + private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default) fun init() { Thread.ofVirtual().name("Download Loop").unstarted(Runnable { @@ -166,7 +167,8 @@ class DownloadService( pending.values.forEach { pending -> pending.removeIf { it.context.image.postId == postId } } - running.values.flatten().filter { p: ImageDownloadRunnable -> p.context.image.postId == postId } + running.values.flatten() + .filter { p: ImageDownloadRunnable -> p.context.image.postId == postId } .forEach { obj: ImageDownloadRunnable -> obj.stop() } } postIds.forEach { @@ -220,7 +222,9 @@ class DownloadService( private fun scheduleForDownload(imageDownloadRunnable: ImageDownloadRunnable) { log.debug("Scheduling a job for ${imageDownloadRunnable.context.image.url}") CompletableFuture.runAsync({ - eventBus.publishEvent(QueueStateEvent(QueueState(runningCount(), pendingCount()))) + coroutineScope.launch { + eventBus.publishEvent(QueueStateEvent(QueueState(runningCount(), pendingCount()))) + } Failsafe.with>(retryPolicyService.buildRetryPolicyForDownload()) .onFailure { try { @@ -243,14 +247,18 @@ class DownloadService( dataTransaction.updateImage(image) }.onComplete { afterJobFinish(imageDownloadRunnable) - eventBus.publishEvent( - QueueStateEvent( - QueueState( - runningCount(), pendingCount() + coroutineScope.launch { + eventBus.publishEvent( + QueueStateEvent( + QueueState( + runningCount(), pendingCount() + ) ) ) - ) - eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataTransaction.countImagesInError()))) + } + coroutineScope.launch { + eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataTransaction.countImagesInError()))) + } log.debug( "Finished downloading ${imageDownloadRunnable.context.image.url}" ) diff --git a/vripper-core/src/main/kotlin/me/vripper/download/ImageDownloadRunnable.kt b/vripper-core/src/main/kotlin/me/vripper/download/ImageDownloadRunnable.kt index 279c1cf..de5f01c 100644 --- a/vripper-core/src/main/kotlin/me/vripper/download/ImageDownloadRunnable.kt +++ b/vripper-core/src/main/kotlin/me/vripper/download/ImageDownloadRunnable.kt @@ -10,8 +10,9 @@ import me.vripper.host.Host import me.vripper.host.ImageMimeType import me.vripper.model.Settings import me.vripper.services.* -import me.vripper.utilities.PathUtils import me.vripper.utilities.PathUtils.getExtension +import me.vripper.utilities.PathUtils.getFileNameWithoutExtension +import me.vripper.utilities.PathUtils.sanitize import net.jodah.failsafe.function.CheckedRunnable import org.koin.core.component.KoinComponent import org.koin.core.component.inject @@ -56,10 +57,6 @@ class ImageDownloadRunnable( val downloadedImage = host.downloadInternal(image.url, context) log.debug("Resolved name for ${image.url}: ${downloadedImage.name}") log.debug("Downloaded image {} to {}", image.url, downloadedImage.path) - val sanitizedFileName = PathUtils.sanitize(downloadedImage.name) - log.debug( - "Sanitizing image name from ${downloadedImage.name} to $sanitizedFileName" - ) synchronized(image.postId.toString().intern()) { val post = dataTransaction.findPostById(context.postId).orElseThrow() val downloadDirectory = Path(post.downloadDirectory, post.folderName).pathString @@ -91,6 +88,7 @@ class ImageDownloadRunnable( downloadDirectory: String, downloadedImage: DownloadedImage, index: Int ) { val existingExtension = getExtension(downloadedImage.name).lowercase() + val fileNameWithoutExtension = getFileNameWithoutExtension(downloadedImage.name) val extension = when (downloadedImage.type) { ImageMimeType.IMAGE_BMP -> "BMP" ImageMimeType.IMAGE_GIF -> "GIF" @@ -99,18 +97,22 @@ class ImageDownloadRunnable( ImageMimeType.IMAGE_WEBP -> "WEBP" } val filename = - if (existingExtension.isBlank()) "${downloadedImage.name}.$extension" else downloadedImage.name + if (existingExtension.isBlank()) "${sanitize(downloadedImage.name)}.$extension" else "${ + sanitize( + fileNameWithoutExtension + ) + }.$extension" try { val downloadDestinationFolder = Path.of(downloadDirectory) Files.createDirectories(downloadDestinationFolder) - val image = downloadDestinationFolder.resolve( - "${ - if (settings.downloadSettings.forceOrder) String.format( - "%03d_", index + 1 - ) else "" - }$filename" - ) - Files.copy(downloadedImage.path, image, StandardCopyOption.REPLACE_EXISTING) + val finalFilename = "${ + if (settings.downloadSettings.forceOrder) String.format( + "%03d_", index + 1 + ) else "" + }$filename" + image.filename = finalFilename + val imageDownloadPath = downloadDestinationFolder.resolve(finalFilename) + Files.copy(downloadedImage.path, imageDownloadPath, StandardCopyOption.REPLACE_EXISTING) } catch (e: Exception) { throw HostException("Failed to rename the image", e) } finally { diff --git a/vripper-core/src/main/kotlin/me/vripper/entities/Image.kt b/vripper-core/src/main/kotlin/me/vripper/entities/Image.kt index 218da9f..7664641 100644 --- a/vripper-core/src/main/kotlin/me/vripper/entities/Image.kt +++ b/vripper-core/src/main/kotlin/me/vripper/entities/Image.kt @@ -13,6 +13,7 @@ data class Image( var size: Long = -1, var downloaded: Long = 0, var status: Status = Status.STOPPED, + var filename: String = "", ) { override fun equals(other: Any?): Boolean { if (this === other) return true diff --git a/vripper-core/src/main/kotlin/me/vripper/entities/LogEntry.kt b/vripper-core/src/main/kotlin/me/vripper/entities/LogEntry.kt index 86fcf54..b764ea5 100644 --- a/vripper-core/src/main/kotlin/me/vripper/entities/LogEntry.kt +++ b/vripper-core/src/main/kotlin/me/vripper/entities/LogEntry.kt @@ -1,13 +1,12 @@ package me.vripper.entities -import com.fasterxml.jackson.annotation.JsonFormat import java.time.LocalDateTime data class LogEntry( val id: Long = -1, val type: Type, val status: Status, - @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss") val time: LocalDateTime = LocalDateTime.now(), + val time: LocalDateTime = LocalDateTime.now(), val message: String, ) { diff --git a/vripper-core/src/main/kotlin/me/vripper/entities/Metadata.kt b/vripper-core/src/main/kotlin/me/vripper/entities/Metadata.kt index 32978e1..fbd0705 100644 --- a/vripper-core/src/main/kotlin/me/vripper/entities/Metadata.kt +++ b/vripper-core/src/main/kotlin/me/vripper/entities/Metadata.kt @@ -1,19 +1,11 @@ package me.vripper.entities -class Metadata { - var postIdRef: Long? = null - var postId: String? = null - var postedBy: String? = null - var resolvedNames = emptyList() +import kotlinx.serialization.Serializable - companion object { - fun from(metadata: Metadata): Metadata { - val copy = Metadata() - copy.postIdRef = metadata.postIdRef - copy.postId = metadata.postId - copy.postedBy = metadata.postedBy - copy.resolvedNames = metadata.resolvedNames - return copy - } - } -} \ No newline at end of file +data class Metadata(val postId: Long, val data: Data) { + @Serializable + data class Data( + val postedBy: String, + val resolvedNames: List + ) +} diff --git a/vripper-core/src/main/kotlin/me/vripper/entities/Post.kt b/vripper-core/src/main/kotlin/me/vripper/entities/Post.kt index b7e26fc..8ff23c7 100644 --- a/vripper-core/src/main/kotlin/me/vripper/entities/Post.kt +++ b/vripper-core/src/main/kotlin/me/vripper/entities/Post.kt @@ -1,6 +1,5 @@ package me.vripper.entities -import com.fasterxml.jackson.annotation.JsonFormat import me.vripper.entities.domain.Status import java.time.LocalDateTime import kotlin.io.path.Path @@ -18,7 +17,7 @@ data class Post( val total: Int, val hosts: Set, val downloadDirectory: String, - @JsonFormat(pattern="yyyy-MM-dd HH:mm:ss") val addedOn: LocalDateTime = LocalDateTime.now(), + val addedOn: LocalDateTime = LocalDateTime.now(), var folderName: String, var status: Status = Status.STOPPED, var done: Int = 0, diff --git a/vripper-core/src/main/kotlin/me/vripper/event/Event.kt b/vripper-core/src/main/kotlin/me/vripper/event/Event.kt index 5b382a8..c20c416 100644 --- a/vripper-core/src/main/kotlin/me/vripper/event/Event.kt +++ b/vripper-core/src/main/kotlin/me/vripper/event/Event.kt @@ -1,9 +1,6 @@ package me.vripper.event -import me.vripper.entities.Image -import me.vripper.entities.LogEntry -import me.vripper.entities.Post -import me.vripper.entities.Thread +import me.vripper.entities.* import me.vripper.model.DownloadSpeed import me.vripper.model.ErrorCount import me.vripper.model.QueueState @@ -21,6 +18,7 @@ data class DownloadSpeedEvent(val downloadSpeed: DownloadSpeed) data class QueueStateEvent(val queueState: QueueState) data class ErrorCountEvent(val errorCount: ErrorCount) data class SettingsUpdateEvent(val settings: Settings) +data class MetadataUpdateEvent(val metadata: Metadata) data class LogCreateEvent(val logEntry: LogEntry) data class LogUpdateEvent(val logEntry: LogEntry) diff --git a/vripper-core/src/main/kotlin/me/vripper/event/EventBus.kt b/vripper-core/src/main/kotlin/me/vripper/event/EventBus.kt index dfdd93a..f9b1f6d 100644 --- a/vripper-core/src/main/kotlin/me/vripper/event/EventBus.kt +++ b/vripper-core/src/main/kotlin/me/vripper/event/EventBus.kt @@ -1,17 +1,14 @@ package me.vripper.event -import reactor.core.publisher.Flux -import reactor.core.publisher.Sinks -import reactor.core.scheduler.Scheduler -import reactor.core.scheduler.Schedulers -import java.util.concurrent.Executors +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.asSharedFlow object EventBus { - private val scheduler: Scheduler = Schedulers.fromExecutor(Executors.newSingleThreadExecutor()) - private val _events = Sinks.many().multicast().onBackpressureBuffer() - val events: Flux = _events.asFlux().publishOn(scheduler) - fun publishEvent(event: Any) { - _events.emitNext(event) { _, _ -> true } + private val _events = MutableSharedFlow() + val events = _events.asSharedFlow() + + suspend fun publishEvent(event: Any) { + _events.emit(event) } } diff --git a/vripper-core/src/main/kotlin/me/vripper/host/Host.kt b/vripper-core/src/main/kotlin/me/vripper/host/Host.kt index 623678c..bf47545 100644 --- a/vripper-core/src/main/kotlin/me/vripper/host/Host.kt +++ b/vripper-core/src/main/kotlin/me/vripper/host/Host.kt @@ -165,6 +165,7 @@ abstract class Host( } } + @OptIn(ExperimentalStdlibApi::class) private fun getImageMimeType(headers: Array
): ImageMimeType? { // first check if content type header exists diff --git a/vripper-core/src/main/kotlin/me/vripper/listeners/OnStartupListener.kt b/vripper-core/src/main/kotlin/me/vripper/listeners/OnStartupListener.kt index fec9ec6..651f617 100644 --- a/vripper-core/src/main/kotlin/me/vripper/listeners/OnStartupListener.kt +++ b/vripper-core/src/main/kotlin/me/vripper/listeners/OnStartupListener.kt @@ -12,6 +12,7 @@ open class OnStartupListener : KoinComponent { private val retryPolicyService: RetryPolicyService by inject() private val vgAuthService: VGAuthService by inject() private val downloadSpeedService: DownloadSpeedService by inject() + private val metadataService: MetadataService by inject() open fun run() { dataTransaction.setDownloadingToStopped() @@ -20,5 +21,6 @@ open class OnStartupListener : KoinComponent { vgAuthService.init() downloadService.init() downloadSpeedService.init() + metadataService.init() } } diff --git a/vripper-core/src/main/kotlin/me/vripper/model/Settings.kt b/vripper-core/src/main/kotlin/me/vripper/model/Settings.kt index f5657f0..5e0af7a 100644 --- a/vripper-core/src/main/kotlin/me/vripper/model/Settings.kt +++ b/vripper-core/src/main/kotlin/me/vripper/model/Settings.kt @@ -1,42 +1,49 @@ package me.vripper.model +import kotlinx.serialization.Serializable + +@Serializable data class Settings( - var connectionSettings: ConnectionSettings = ConnectionSettings(), - var downloadSettings: DownloadSettings = DownloadSettings(), - var viperSettings: ViperSettings = ViperSettings(), - var systemSettings: SystemSettings = SystemSettings() + val connectionSettings: ConnectionSettings = ConnectionSettings(), + val downloadSettings: DownloadSettings = DownloadSettings(), + val viperSettings: ViperSettings = ViperSettings(), + val systemSettings: SystemSettings = SystemSettings() ) +@Serializable data class ViperSettings( - var login: Boolean = false, - var username: String = "", - var password: String = "", - var thanks: Boolean = false, - var host: String = "https://vipergirls.to", + val login: Boolean = false, + val username: String = "", + val password: String = "", + val thanks: Boolean = false, + val host: String = "https://vipergirls.to", ) +@Serializable data class DownloadSettings( - var downloadPath: String = System.getProperty("user.home"), - var autoStart: Boolean = true, - var autoQueueThreshold: Int = 1, - var forceOrder: Boolean = false, - var forumSubDirectory: Boolean = false, - var threadSubLocation: Boolean = false, - var clearCompleted: Boolean = false, - var appendPostId: Boolean = false + val downloadPath: String = System.getProperty("user.home"), + val autoStart: Boolean = true, + val autoQueueThreshold: Int = 1, + val forceOrder: Boolean = false, + val forumSubDirectory: Boolean = false, + val threadSubLocation: Boolean = false, + val clearCompleted: Boolean = false, + val appendPostId: Boolean = false ) +@Serializable data class ConnectionSettings( - var maxConcurrentPerHost: Int = 2, - var maxGlobalConcurrent: Int = 0, - var timeout: Long = 30, - var maxAttempts: Int = 3, + val maxConcurrentPerHost: Int = 2, + val maxGlobalConcurrent: Int = 0, + val timeout: Long = 30, + val maxAttempts: Int = 3, ) +@Serializable data class SystemSettings( - var tempPath: String = System.getProperty("java.io.tmpdir"), - var cachePath: String = System.getProperty("java.io.tmpdir"), - var enableClipboardMonitoring: Boolean = false, - var clipboardPollingRate: Int = 500, - var maxEventLog: Int = 1_000, + val tempPath: String = System.getProperty("java.io.tmpdir"), + val cachePath: String = System.getProperty("java.io.tmpdir"), + val enableClipboardMonitoring: Boolean = false, + val clipboardPollingRate: Int = 500, + val maxEventLog: Int = 1_000, ) diff --git a/vripper-core/src/main/kotlin/me/vripper/repositories/MetadataRepository.kt b/vripper-core/src/main/kotlin/me/vripper/repositories/MetadataRepository.kt index c36e92c..75b02b6 100644 --- a/vripper-core/src/main/kotlin/me/vripper/repositories/MetadataRepository.kt +++ b/vripper-core/src/main/kotlin/me/vripper/repositories/MetadataRepository.kt @@ -7,4 +7,5 @@ interface MetadataRepository { fun save(metadata: Metadata): Metadata fun findByPostId(postId: Long): Optional fun deleteByPostId(postId: Long): Int + fun deleteAllByPostId(postIds: List) } \ No newline at end of file diff --git a/vripper-core/src/main/kotlin/me/vripper/repositories/impl/ImageRepositoryImpl.kt b/vripper-core/src/main/kotlin/me/vripper/repositories/impl/ImageRepositoryImpl.kt index a796770..7e5fdfa 100644 --- a/vripper-core/src/main/kotlin/me/vripper/repositories/impl/ImageRepositoryImpl.kt +++ b/vripper-core/src/main/kotlin/me/vripper/repositories/impl/ImageRepositoryImpl.kt @@ -22,6 +22,7 @@ class ImageRepositoryImpl : ImageRepository { it[url] = image.url it[thumbUrl] = image.thumbUrl it[postIdRef] = image.postIdRef + it[filename] = image.filename }.value return image.copy(id = id) } @@ -37,6 +38,7 @@ class ImageRepositoryImpl : ImageRepository { this[ImageTable.url] = it.url this[ImageTable.thumbUrl] = it.thumbUrl this[ImageTable.postIdRef] = it.postIdRef + this[ImageTable.filename] = it.filename } } @@ -98,6 +100,7 @@ class ImageRepositoryImpl : ImageRepository { it[status] = image.status.name it[downloaded] = image.downloaded it[size] = image.size + it[filename] = image.filename } } @@ -113,6 +116,7 @@ class ImageRepositoryImpl : ImageRepository { this[ImageTable.url] = it.url this[ImageTable.thumbUrl] = it.thumbUrl this[ImageTable.postIdRef] = it.postIdRef + this[ImageTable.filename] = it.filename } } @@ -150,6 +154,7 @@ class ImageRepositoryImpl : ImageRepository { val total = resultRow[ImageTable.size] val status = Status.valueOf(resultRow[ImageTable.status]) val postIdRef = resultRow[ImageTable.postIdRef] + val filename = resultRow[ImageTable.filename] return Image( id, postId, @@ -160,7 +165,8 @@ class ImageRepositoryImpl : ImageRepository { postIdRef, total, current, - status + status, + filename ) } } diff --git a/vripper-core/src/main/kotlin/me/vripper/repositories/impl/MetadataRepositoryImpl.kt b/vripper-core/src/main/kotlin/me/vripper/repositories/impl/MetadataRepositoryImpl.kt index 4ec2cda..2b59d94 100644 --- a/vripper-core/src/main/kotlin/me/vripper/repositories/impl/MetadataRepositoryImpl.kt +++ b/vripper-core/src/main/kotlin/me/vripper/repositories/impl/MetadataRepositoryImpl.kt @@ -1,20 +1,73 @@ package me.vripper.repositories.impl +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json import me.vripper.entities.Metadata import me.vripper.repositories.MetadataRepository +import me.vripper.tables.MetadataTable +import org.jetbrains.exposed.sql.ResultRow +import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq +import org.jetbrains.exposed.sql.deleteWhere +import org.jetbrains.exposed.sql.insert +import org.jetbrains.exposed.sql.select +import org.jetbrains.exposed.sql.transactions.TransactionManager +import java.sql.Connection import java.util.* class MetadataRepositoryImpl: MetadataRepository { override fun save(metadata: Metadata): Metadata { + MetadataTable.insert { + it[postId] = metadata.postId + it[data] = Json.encodeToString(metadata.data) + } + return metadata } override fun findByPostId(postId: Long): Optional { - return Optional.empty() + + val result = MetadataTable.select { + MetadataTable.postId eq postId + }.map(::transform) + + return if (result.isEmpty()) { + Optional.empty() + } else { + Optional.of(result.first()) + } + } + + private fun transform(row: ResultRow): Metadata { + val id = row[MetadataTable.postId] + val data = Json.decodeFromString(row[MetadataTable.data]) as Metadata.Data + return Metadata(id, data) } override fun deleteByPostId(postId: Long): Int { - return 0 + return MetadataTable.deleteWhere { MetadataTable.postId eq postId } + } + + override fun deleteAllByPostId(postIds: List) { + val conn = TransactionManager.current().connection.connection as Connection + conn.prepareStatement("CREATE LOCAL TEMPORARY TABLE METADATA_DELETE(POST_ID BIGINT PRIMARY KEY)") + .use { + it.execute() + } + + conn.prepareStatement("INSERT INTO METADATA_DELETE VALUES ( ? )").use { ps -> + postIds.forEach { + ps.setLong(1, it) + ps.addBatch() + } + ps.executeBatch() + } + + conn.prepareStatement("DELETE FROM METADATA WHERE POST_ID IN (SELECT POST_ID FROM METADATA_DELETE)") + .use { + it.execute() + } + + conn.prepareStatement("TRUNCATE TABLE METADATA_DELETE") } } diff --git a/vripper-core/src/main/kotlin/me/vripper/services/AppEndpointService.kt b/vripper-core/src/main/kotlin/me/vripper/services/AppEndpointService.kt index 400ce42..817164f 100644 --- a/vripper-core/src/main/kotlin/me/vripper/services/AppEndpointService.kt +++ b/vripper-core/src/main/kotlin/me/vripper/services/AppEndpointService.kt @@ -144,14 +144,19 @@ class AppEndpointService( dataTransaction.deleteAllLogs() } - fun rename(postId: Long, name: String) { + fun rename(postId: Long, newName: String) { CompletableFuture.runAsync({ synchronized(postId.toString().intern()) { dataTransaction.findPostByPostId(postId).ifPresent { post -> if (Path(post.downloadDirectory, post.folderName).exists()) { - PathUtils.rename(post.downloadDirectory, post.folderName, name) + PathUtils.rename( + dataTransaction.findImagesByPostId(postId), + post.downloadDirectory, + post.folderName, + newName + ) } - post.folderName = name + post.folderName = PathUtils.sanitize(newName) dataTransaction.updatePost(post) } } diff --git a/vripper-core/src/main/kotlin/me/vripper/services/DataTransaction.kt b/vripper-core/src/main/kotlin/me/vripper/services/DataTransaction.kt index 4ea0266..ce96366 100644 --- a/vripper-core/src/main/kotlin/me/vripper/services/DataTransaction.kt +++ b/vripper-core/src/main/kotlin/me/vripper/services/DataTransaction.kt @@ -1,5 +1,9 @@ package me.vripper.services +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch import me.vripper.entities.* import me.vripper.entities.domain.Status import me.vripper.event.* @@ -22,33 +26,45 @@ class DataTransaction( private val eventBus: EventBus, ) { + private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + private fun save(posts: List): List { return transaction { postDownloadStateRepository.save(posts) } } fun updatePosts(posts: List) { transaction { postDownloadStateRepository.update(posts) } - eventBus.publishEvent(PostUpdateEvent(posts)) + coroutineScope.launch { + eventBus.publishEvent(PostUpdateEvent(posts)) + } } fun updatePost(post: Post) { transaction { postDownloadStateRepository.update(post) } - eventBus.publishEvent(PostUpdateEvent(listOf(post))) + coroutineScope.launch { + eventBus.publishEvent(PostUpdateEvent(listOf(post))) + } } fun save(thread: Thread) { val savedThread = transaction { threadRepository.save(thread) } - eventBus.publishEvent(ThreadCreateEvent(savedThread)) + coroutineScope.launch { + eventBus.publishEvent(ThreadCreateEvent(savedThread)) + } } fun updateImages(images: List) { transaction { imageRepository.update(images) } - eventBus.publishEvent(ImageEvent(images)) + coroutineScope.launch { + eventBus.publishEvent(ImageEvent(images)) + } } fun updateImage(image: Image) { transaction { imageRepository.update(image) } - eventBus.publishEvent(ImageEvent(listOf(image))) + coroutineScope.launch { + eventBus.publishEvent(ImageEvent(listOf(image))) + } } fun exists(postId: Long): Boolean { @@ -104,7 +120,9 @@ class DataTransaction( } savedPosts } - eventBus.publishEvent(PostCreateEvent(savedPosts)) + coroutineScope.launch { + eventBus.publishEvent(PostCreateEvent(savedPosts)) + } return savedPosts } @@ -146,21 +164,25 @@ class DataTransaction( private fun remove(postIds: List) { transaction { + metadataRepository.deleteAllByPostId(postIds) imageRepository.deleteAllByPostId(postIds) postDownloadStateRepository.deleteAll(postIds) sortPostsByRank() } - - eventBus.publishEvent(PostDeleteEvent(postIds = postIds)) - eventBus.publishEvent(ErrorCountEvent(ErrorCount(countImagesInError()))) + coroutineScope.launch { + eventBus.publishEvent(PostDeleteEvent(postIds = postIds)) + } + coroutineScope.launch { + eventBus.publishEvent(ErrorCountEvent(ErrorCount(countImagesInError()))) + } } fun removeThread(threadId: Long) { transaction { threadRepository.deleteByThreadId(threadId) } - - eventBus.publishEvent(ThreadDeleteEvent(threadId)) - + coroutineScope.launch { + eventBus.publishEvent(ThreadDeleteEvent(threadId)) + } } fun clearCompleted(): List { @@ -181,17 +203,18 @@ class DataTransaction( transaction { imageRepository.stopByPostIdAndIsNotCompleted(postId) } } - @Synchronized - fun setMetadata(post: Post, metadata: Metadata) { - if (metadataRepository.findByPostId(post.postId).isEmpty) { - metadata.postIdRef = post.id - metadataRepository.save(metadata) + fun saveMetadata(metadata: Metadata) { + transaction { metadataRepository.save(metadata) } + coroutineScope.launch { + eventBus.publishEvent(MetadataUpdateEvent(metadata)) } } fun clearQueueLinks() { transaction { threadRepository.deleteAll() } - eventBus.publishEvent(ThreadClearEvent()) + coroutineScope.launch { + eventBus.publishEvent(ThreadClearEvent()) + } } @Synchronized @@ -255,14 +278,20 @@ class DataTransaction( val deleted = logRepository.deleteOldest() Pair(saved, deleted) } - eventBus.publishEvent(LogCreateEvent(pair.first)) - eventBus.publishEvent(LogDeleteEvent(pair.second)) + coroutineScope.launch { + eventBus.publishEvent(LogCreateEvent(pair.first)) + } + coroutineScope.launch { + eventBus.publishEvent(LogDeleteEvent(pair.second)) + } return pair.first } fun updateLog(logEntry: LogEntry) { transaction { logRepository.update(logEntry) } - eventBus.publishEvent(LogUpdateEvent(logEntry)) + coroutineScope.launch { + eventBus.publishEvent(LogUpdateEvent(logEntry)) + } } fun deleteAllLogs() { @@ -276,4 +305,8 @@ class DataTransaction( fun findAllNonCompletedPostIds(): List { return transaction { postDownloadStateRepository.findAllNonCompletedPostIds() } } + + fun findMetadataByPostId(postId: Long): Optional { + return transaction { metadataRepository.findByPostId(postId) } + } } \ No newline at end of file diff --git a/vripper-core/src/main/kotlin/me/vripper/services/DownloadSpeedService.kt b/vripper-core/src/main/kotlin/me/vripper/services/DownloadSpeedService.kt index 16e5d66..bba50e7 100644 --- a/vripper-core/src/main/kotlin/me/vripper/services/DownloadSpeedService.kt +++ b/vripper-core/src/main/kotlin/me/vripper/services/DownloadSpeedService.kt @@ -1,6 +1,7 @@ package me.vripper.services import kotlinx.coroutines.* +import kotlinx.coroutines.flow.filterIsInstance import me.vripper.event.DownloadSpeedEvent import me.vripper.event.EventBus import me.vripper.event.QueueStateEvent @@ -21,7 +22,8 @@ class DownloadSpeedService( fun init() { coroutineScope.launch { - eventBus.events.ofType(QueueStateEvent::class.java).subscribe { + eventBus.events.filterIsInstance(QueueStateEvent::class).collect { + coroutineContext.ensureActive() if (it.queueState.running + it.queueState.remaining > 0) { if (job == null || job?.isActive == false) { job = coroutineScope.launch { diff --git a/vripper-core/src/main/kotlin/me/vripper/services/HTTPService.kt b/vripper-core/src/main/kotlin/me/vripper/services/HTTPService.kt index b112eb4..ac8aed7 100644 --- a/vripper-core/src/main/kotlin/me/vripper/services/HTTPService.kt +++ b/vripper-core/src/main/kotlin/me/vripper/services/HTTPService.kt @@ -1,6 +1,7 @@ package me.vripper.services import kotlinx.coroutines.* +import kotlinx.coroutines.flow.filterIsInstance import me.vripper.event.EventBus import me.vripper.event.SettingsUpdateEvent import org.apache.hc.client5.http.config.ConnectionConfig @@ -17,7 +18,7 @@ import org.apache.hc.core5.util.TimeValue import org.apache.hc.core5.util.Timeout class HTTPService( - val eventBus: EventBus, + private val eventBus: EventBus, settingsService: SettingsService ) { @@ -43,20 +44,22 @@ class HTTPService( pcm.closeIdle(TimeValue.ofSeconds(60)) delay(15000) } - eventBus - .events - .ofType(SettingsUpdateEvent::class.java) - .subscribe { - if (connectionTimeout != it.settings.connectionSettings.timeout) { - connectionTimeout = it.settings.connectionSettings.timeout - client.close() - pcm.close() - buildRequestConfig() - buildConnectionConfig() - buildConnectionPool() - buildClientBuilder() + coroutineScope.launch { + eventBus + .events + .filterIsInstance(SettingsUpdateEvent::class) + .collect { + if (connectionTimeout != it.settings.connectionSettings.timeout) { + connectionTimeout = it.settings.connectionSettings.timeout + client.close() + pcm.close() + buildRequestConfig() + buildConnectionConfig() + buildConnectionPool() + buildClientBuilder() + } } - } + } } private fun buildConnectionPool() { diff --git a/vripper-core/src/main/kotlin/me/vripper/services/MetadataService.kt b/vripper-core/src/main/kotlin/me/vripper/services/MetadataService.kt new file mode 100644 index 0000000..81d7ad6 --- /dev/null +++ b/vripper-core/src/main/kotlin/me/vripper/services/MetadataService.kt @@ -0,0 +1,109 @@ +package me.vripper.services + +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit +import me.vripper.entities.Metadata +import me.vripper.exception.DownloadException +import me.vripper.exception.VripperException +import me.vripper.utilities.HtmlUtils +import me.vripper.utilities.XpathUtils +import org.apache.hc.client5.http.classic.methods.HttpGet +import org.apache.hc.core5.net.URIBuilder +import org.w3c.dom.Node +import java.util.concurrent.atomic.AtomicBoolean +import java.util.stream.Collectors + + +class MetadataService( + private val httpService: HTTPService, + private val settingsService: SettingsService, + private val dataTransaction: DataTransaction, + private val vgAuthService: VGAuthService, +) { + + private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private val requestSemaphore = Semaphore(6) + private val dictionary: List = + mutableListOf("download", "link", "rapidgator", "filefactory", "filefox") + + fun init() { + dataTransaction.findAllPosts() + .filter { dataTransaction.findMetadataByPostId(it.postId).isEmpty }.map { it.postId } + .forEach(::fetchMetadata) + } + + fun fetchMetadata(postId: Long) { + coroutineScope.launch { + requestSemaphore.withPermit { + val httpGet = + HttpGet(URIBuilder(settingsService.settings.viperSettings.host + "/threads/").also { + it.setParameter( + "p", postId.toString() + ) + }.build()) + + val metadata = httpService.client.execute(httpGet, vgAuthService.context) { + if (it.code / 100 != 2) { + throw DownloadException("Unexpected response code '${it.code}' for $httpGet") + } + val document = HtmlUtils.clean(it.entity.content) + val postNode: Node = XpathUtils.getAsNode( + document, + "//li[@id='post_$postId']/div[contains(@class, 'postdetails')]", + + ) ?: throw VripperException("Unable to find post #'$postId'") + + val postedBy: String = XpathUtils.getAsNode( + postNode, + "./div[contains(@class, 'userinfo')]//a[contains(@class, 'username')]//font" + )?.textContent?.trim() + ?: throw VripperException("Unable to find the poster for post #'$postId'") + + + val node: Node = XpathUtils.getAsNode( + document, java.lang.String.format("//div[@id='post_message_%s']", postId) + ) ?: throw VripperException("Unable to locate post content") + val titles = findTitleInContent(node) + Metadata(postId, Metadata.Data(postedBy, titles)) + } + dataTransaction.saveMetadata(metadata) + } + } + } + + private fun findTitleInContent(node: Node): List { + val altTitle: MutableList = mutableListOf() + findTitle(node, altTitle, AtomicBoolean(true)) + return altTitle.stream().distinct().collect(Collectors.toList()) + } + + private fun findTitle(node: Node, altTitle: MutableList, keepGoing: AtomicBoolean) { + if (!keepGoing.get()) { + return + } + if (node.nodeName == "a" || node.nodeName == "img") { + keepGoing.set(false) + return + } + if (node.nodeType == Node.ELEMENT_NODE) { + for (i in 0 until node.childNodes.length) { + val item = node.childNodes.item(i) + findTitle(item, altTitle, keepGoing) + if (!keepGoing.get()) { + return + } + } + } else if (node.nodeType == Node.TEXT_NODE) { + val text = node.textContent.trim() + if (text.isNotBlank() && dictionary.stream().noneMatch { e -> + text.lowercase().contains(e.lowercase()) + }) { + altTitle.add(text) + } + } + } +} \ No newline at end of file diff --git a/vripper-core/src/main/kotlin/me/vripper/services/RetryPolicyService.kt b/vripper-core/src/main/kotlin/me/vripper/services/RetryPolicyService.kt index dff1e23..0133569 100644 --- a/vripper-core/src/main/kotlin/me/vripper/services/RetryPolicyService.kt +++ b/vripper-core/src/main/kotlin/me/vripper/services/RetryPolicyService.kt @@ -1,5 +1,10 @@ package me.vripper.services +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.filterIsInstance +import kotlinx.coroutines.launch import me.vripper.event.EventBus import me.vripper.event.SettingsUpdateEvent import net.jodah.failsafe.RetryPolicy @@ -11,14 +16,16 @@ class RetryPolicyService( ) { private val log by me.vripper.delegate.LoggerDelegate() private var maxAttempts: Int = settingsService.settings.connectionSettings.maxAttempts + private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default) fun init() { - eventBus.events.ofType(SettingsUpdateEvent::class.java).subscribe { - if (maxAttempts != it.settings.connectionSettings.maxAttempts) { - maxAttempts = it.settings.connectionSettings.maxAttempts + coroutineScope.launch { + eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collect { + if (maxAttempts != it.settings.connectionSettings.maxAttempts) { + maxAttempts = it.settings.connectionSettings.maxAttempts + } } } - } fun buildRetryPolicyForDownload(): RetryPolicy { diff --git a/vripper-core/src/main/kotlin/me/vripper/services/SettingsService.kt b/vripper-core/src/main/kotlin/me/vripper/services/SettingsService.kt index ea0c425..de0e27a 100644 --- a/vripper-core/src/main/kotlin/me/vripper/services/SettingsService.kt +++ b/vripper-core/src/main/kotlin/me/vripper/services/SettingsService.kt @@ -1,10 +1,12 @@ package me.vripper.services -import com.fasterxml.jackson.databind.DeserializationFeature -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory -import com.fasterxml.jackson.module.kotlin.readValue -import com.fasterxml.jackson.module.kotlin.registerKotlinModule +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.decodeFromStream import me.vripper.event.EventBus import me.vripper.event.SettingsUpdateEvent import me.vripper.exception.ValidationException @@ -13,42 +15,45 @@ import me.vripper.utilities.ApplicationProperties.VRIPPER_DIR import org.apache.commons.codec.digest.DigestUtils import java.io.FileWriter import java.nio.file.* +import kotlin.io.path.readText class SettingsService(private val eventBus: EventBus) { private val log by me.vripper.delegate.LoggerDelegate() - private val configPath = VRIPPER_DIR.resolve("config.yml") + private val configPath = VRIPPER_DIR.resolve("config.json") private val customProxiesPath = VRIPPER_DIR.resolve("proxies.json") - private val om = ObjectMapper(YAMLFactory()) private val proxies: MutableSet = HashSet() + private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default) - + private val json = Json { + encodeDefaults = true + prettyPrint = true + } var settings = Settings() init { - om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - .findAndRegisterModules().registerKotlinModule() - init() } private fun init() { loadViperProxies() restore() - eventBus.publishEvent(SettingsUpdateEvent(settings)) + coroutineScope.launch { + eventBus.publishEvent(SettingsUpdateEvent(settings)) + } } private fun loadViperProxies() { try { SettingsService::class.java.getResourceAsStream("/proxies.json")?.use { - val defaultProxies: List = om.readValue(it) + val defaultProxies: List = json.decodeFromStream(it) val customProxies: List = if (customProxiesPath.toFile() .exists() && Files.isRegularFile(customProxiesPath) ) { try { - om.readValue( - customProxiesPath.toFile() + json.decodeFromString( + customProxiesPath.readText() ) } catch (e: Exception) { emptyList() @@ -81,36 +86,35 @@ class SettingsService(private val eventBus: EventBus) { } fun newSettings(settings: Settings) { - if (settings.viperSettings.login) { + check(settings) + val viperSettings = if (settings.viperSettings.login) { if (this.settings.viperSettings.password != settings.viperSettings.password) { - settings.viperSettings.password = - DigestUtils.md5Hex(settings.viperSettings.password) + settings.viperSettings.copy(password = DigestUtils.md5Hex(settings.viperSettings.password)) + } else { + settings.viperSettings } } else { - settings.viperSettings.username = "" - settings.viperSettings.password = "" - settings.viperSettings.thanks = false - settings.viperSettings.login = false + settings.viperSettings.copy(username = "", password = "", thanks = false, login = false) } - check(settings) - this.settings = settings + this.settings = settings.copy(viperSettings = viperSettings) save() - - eventBus.publishEvent(SettingsUpdateEvent(settings)) - + coroutineScope.launch { + eventBus.publishEvent(SettingsUpdateEvent(settings)) + } } private fun restore() { try { if (configPath.toFile().exists()) { - settings = om.readValue(configPath.toFile()) + settings = json.decodeFromString(configPath.readText()) } } catch (e: Exception) { log.error("Failed restore user settings", e) settings = Settings() } if (!proxies.contains(settings.viperSettings.host)) { - settings.viperSettings.host = "https://vipergirls.to" + val viperSetting = settings.viperSettings.copy(host = "https://vipergirls.to") + settings = settings.copy(viperSettings = viperSetting) } try { check(settings) @@ -123,9 +127,9 @@ class SettingsService(private val eventBus: EventBus) { fun save() { try { - Files.write( + Files.writeString( configPath, - om.writeValueAsBytes(settings), + json.encodeToString(settings), StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING, diff --git a/vripper-core/src/main/kotlin/me/vripper/services/ThreadCacheService.kt b/vripper-core/src/main/kotlin/me/vripper/services/ThreadCacheService.kt index dd253db..13ff7ff 100644 --- a/vripper-core/src/main/kotlin/me/vripper/services/ThreadCacheService.kt +++ b/vripper-core/src/main/kotlin/me/vripper/services/ThreadCacheService.kt @@ -2,6 +2,11 @@ package me.vripper.services import com.github.benmanes.caffeine.cache.Caffeine import com.github.benmanes.caffeine.cache.LoadingCache +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.filterIsInstance +import kotlinx.coroutines.launch import me.vripper.event.EventBus import me.vripper.event.SettingsUpdateEvent import me.vripper.model.ThreadItem @@ -12,14 +17,18 @@ import java.util.concurrent.TimeUnit class ThreadCacheService(val eventBus: EventBus) { + private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + fun init() { - eventBus.events.ofType(SettingsUpdateEvent::class.java).subscribe { - cache.invalidateAll() + coroutineScope.launch { + eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collect { + cache.invalidateAll() + } } } private val cache: LoadingCache = - Caffeine.newBuilder().expireAfterWrite(5, TimeUnit.MINUTES).build { threadId -> + Caffeine.newBuilder().expireAfterWrite(20, TimeUnit.MINUTES).build { threadId -> ThreadLookupAPIParser(threadId).parse() } diff --git a/vripper-core/src/main/kotlin/me/vripper/services/VGAuthService.kt b/vripper-core/src/main/kotlin/me/vripper/services/VGAuthService.kt index 1fceeb4..b047f83 100644 --- a/vripper-core/src/main/kotlin/me/vripper/services/VGAuthService.kt +++ b/vripper-core/src/main/kotlin/me/vripper/services/VGAuthService.kt @@ -1,5 +1,10 @@ package me.vripper.services +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.filterIsInstance +import kotlinx.coroutines.launch import me.vripper.entities.Post import me.vripper.event.EventBus import me.vripper.event.SettingsUpdateEvent @@ -18,8 +23,11 @@ import org.apache.hc.core5.http.message.BasicNameValuePair import java.util.concurrent.CompletableFuture class VGAuthService( - private val cm: HTTPService, private val settingsService: SettingsService, private val eventBus: EventBus + private val cm: HTTPService, + private val settingsService: SettingsService, + private val eventBus: EventBus ) { + private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default) private val log by me.vripper.delegate.LoggerDelegate() val context: HttpClientContext = HttpClientContext.create() var loggedUser = "" @@ -28,8 +36,10 @@ class VGAuthService( fun init() { context.cookieStore = BasicCookieStore() - eventBus.events.ofType(SettingsUpdateEvent::class.java).subscribe { - authenticate(it.settings) + coroutineScope.launch { + eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collect { + authenticate(it.settings) + } } authenticate(settingsService.settings) } @@ -40,7 +50,9 @@ class VGAuthService( log.debug("Authentication option is disabled") context.cookieStore.clear() loggedUser = "" - eventBus.publishEvent(VGUserLoginEvent(loggedUser)) + coroutineScope.launch { + eventBus.publishEvent(VGUserLoginEvent(loggedUser)) + } return } @@ -51,24 +63,26 @@ class VGAuthService( context.cookieStore.clear() loggedUser = "" - eventBus.publishEvent(VGUserLoginEvent(loggedUser)) + coroutineScope.launch { + eventBus.publishEvent(VGUserLoginEvent(loggedUser)) + } return } val postAuth = HttpPost(settings.viperSettings.host + "/login.php?do=login").also { - it.entity = UrlEncodedFormEntity( - listOf( - BasicNameValuePair("vb_login_username", username), - BasicNameValuePair("cookieuser", "1"), - BasicNameValuePair("do", "login"), - BasicNameValuePair("vb_login_md5password", password) - ) + it.entity = UrlEncodedFormEntity( + listOf( + BasicNameValuePair("vb_login_username", username), + BasicNameValuePair("cookieuser", "1"), + BasicNameValuePair("do", "login"), + BasicNameValuePair("vb_login_md5password", password) ) - it.addHeader("Referer", settings.viperSettings.host) - it.addHeader( - "Host", settings.viperSettings.host.replace("https://", "").replace("http://", "") - ) - } + ) + it.addHeader("Referer", settings.viperSettings.host) + it.addHeader( + "Host", settings.viperSettings.host.replace("https://", "").replace("http://", "") + ) + } try { cm.client.execute(postAuth, context) { response -> @@ -81,7 +95,8 @@ class VGAuthService( if (context.cookieStore.cookies.stream().map { obj: Cookie -> obj.name } .noneMatch { e: String -> e == "vg_userid" }) { log.error( - "Failed to authenticate user with {}, missing vg_userid cookie", settings.viperSettings.host + "Failed to authenticate user with {}, missing vg_userid cookie", + settings.viperSettings.host ) return } @@ -89,7 +104,9 @@ class VGAuthService( context.cookieStore.clear() loggedUser = "" - eventBus.publishEvent(VGUserLoginEvent(loggedUser)) + coroutineScope.launch { + eventBus.publishEvent(VGUserLoginEvent(loggedUser)) + } log.error( "Failed to authenticate user with " + settings.viperSettings.host, e @@ -100,11 +117,14 @@ class VGAuthService( loggedUser = username log.info(String.format("Authenticated: %s", username)) - eventBus.publishEvent(VGUserLoginEvent(loggedUser)) - + coroutineScope.launch { + eventBus.publishEvent(VGUserLoginEvent(loggedUser)) + } } fun leaveThanks(post: Post) { - CompletableFuture.runAsync(LeaveThanksRunnable(post, authenticated, context), GLOBAL_EXECUTOR) + CompletableFuture.runAsync( + LeaveThanksRunnable(post, authenticated, context), GLOBAL_EXECUTOR + ) } } \ No newline at end of file diff --git a/vripper-core/src/main/kotlin/me/vripper/tables/ImageTable.kt b/vripper-core/src/main/kotlin/me/vripper/tables/ImageTable.kt index 1cb4505..bcba2ff 100644 --- a/vripper-core/src/main/kotlin/me/vripper/tables/ImageTable.kt +++ b/vripper-core/src/main/kotlin/me/vripper/tables/ImageTable.kt @@ -8,6 +8,7 @@ object ImageTable : LongIdTable(name = "IMAGE", columnName = "ID") { val index = integer("INDEX") val postId = long("POST_ID") val status = varchar("STATUS", 15) + val filename = varchar("FILENAME", 260) val size = long("SIZE") val url = varchar("URL", 200) val thumbUrl = varchar("THUMB_URL", 200) diff --git a/vripper-core/src/main/kotlin/me/vripper/tables/MetadataTable.kt b/vripper-core/src/main/kotlin/me/vripper/tables/MetadataTable.kt new file mode 100644 index 0000000..7d9e5a6 --- /dev/null +++ b/vripper-core/src/main/kotlin/me/vripper/tables/MetadataTable.kt @@ -0,0 +1,8 @@ +package me.vripper.tables + +import org.jetbrains.exposed.sql.Table + +object MetadataTable : Table(name = "METADATA") { + val postId = long("POST_ID") + val data = varchar("DATA", 1_000_000) +} \ No newline at end of file diff --git a/vripper-core/src/main/kotlin/me/vripper/tasks/AddPostRunnable.kt b/vripper-core/src/main/kotlin/me/vripper/tasks/AddPostRunnable.kt index 09ac695..f74d68e 100644 --- a/vripper-core/src/main/kotlin/me/vripper/tasks/AddPostRunnable.kt +++ b/vripper-core/src/main/kotlin/me/vripper/tasks/AddPostRunnable.kt @@ -28,64 +28,28 @@ class AddPostRunnable(private val items: List) : KoinComponent, Ru private val retryPolicyService: RetryPolicyService by inject() private val downloadService: DownloadService by inject() private val cacheService: ThreadCacheService by inject() -// private val logEntry: LogEntry - - init { -// logEntry = dataTransaction.saveLog( -// LogEntry( -// type = LogEntry.Type.POST, status = PENDING, message = "Processing $link" -// ) -// ) - } + private val metadataService: MetadataService by inject() override fun run() { try { Tasks.increment() val toProcess = mutableListOf() for ((threadId, postId) in items) { -// dataTransaction.updateLog(logEntry.copy(status = PROCESSING)) if (dataTransaction.exists(postId)) { log.warn(String.format("skipping %s, already loaded", postId)) continue } val threadItem = cacheService[threadId] -// val postItem: PostItem = try { val postItem: PostItem = threadItem.postItemList.find { it.postId == postId } ?: parse(postId, threadId) -// } catch (e: PostParseException) { -// val error = String.format("parsing failed for gallery %s", link) -// log.error(error, e) -// dataTransaction.updateLog( -// logEntry.copy( -// status = ERROR, message = """ -// $error -// ${e.formatToString()} -// """.trimIndent() -// ) -// ) -// return@map -// } -// if (postItem.imageItemList.isEmpty()) { -// val error = "Post $link contains no images to download" -// log.error(error) -// dataTransaction.updateLog(logEntry.copy(status = ERROR, message = error)) -// return@map -// } - - -// dataTransaction.updateLog( -// logEntry.copy( -// status = DONE, message = String.format( -// "Post $link has been successfully added to download queue" -// ) -// ) -// ) toProcess.add(postItem) } val posts = dataTransaction.newPosts(toProcess.toList()) -// metadataService.startFetchingMetadata(post) + posts.forEach { + metadataService.fetchMetadata(it.postId) + } if (settingsService.settings.downloadSettings.autoStart) { log.debug("Auto start downloads option is enabled") downloadService.restartAll(posts) @@ -93,14 +57,6 @@ class AddPostRunnable(private val items: List) : KoinComponent, Ru } catch (e: Exception) { val error = String.format("Error when adding galleries") log.error(error, e) -// dataTransaction.updateLog( -// logEntry.copy( -// status = ERROR, message = """ -// $error -// ${e.formatToString()} -// """.trimIndent() -// ) -// ) } finally { Tasks.decrement() } diff --git a/vripper-core/src/main/kotlin/me/vripper/utilities/PathUtils.kt b/vripper-core/src/main/kotlin/me/vripper/utilities/PathUtils.kt index ef51ecf..fd19ccc 100644 --- a/vripper-core/src/main/kotlin/me/vripper/utilities/PathUtils.kt +++ b/vripper-core/src/main/kotlin/me/vripper/utilities/PathUtils.kt @@ -1,5 +1,6 @@ package me.vripper.utilities +import me.vripper.entities.Image import me.vripper.exception.RenameException import me.vripper.model.Settings import java.io.IOException @@ -7,6 +8,7 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardCopyOption import kotlin.io.path.Path +import kotlin.io.path.listDirectoryEntries object PathUtils { @@ -28,15 +30,32 @@ object PathUtils { } @Throws(RenameException::class) - fun rename(downloadDirectory: String, oldFolder: String, newFolder: String) { + fun rename( + imageList: List, + downloadDirectory: String, + oldFolder: String, + newFolder: String + ) { val currentDownloadDirectory = Path(downloadDirectory, oldFolder) val newDownloadDirectory = Path(downloadDirectory, sanitize(newFolder)) + if (currentDownloadDirectory == newDownloadDirectory) { + return + } try { - Files.move( - currentDownloadDirectory, - newDownloadDirectory, - StandardCopyOption.ATOMIC_MOVE - ) + Files.createDirectories(newDownloadDirectory) + imageList.filter { it.filename.isNotBlank() }.forEach { + Files.move( + currentDownloadDirectory.resolve(it.filename), + newDownloadDirectory.resolve(it.filename), + StandardCopyOption.ATOMIC_MOVE + ) + } + if (currentDownloadDirectory.listDirectoryEntries().isEmpty()) { + try { + Files.delete(currentDownloadDirectory) + } catch (ignored: Exception) { + } + } } catch (e: IOException) { throw RenameException( String.format( diff --git a/vripper-core/src/main/kotlin/me/vripper/utilities/Tasks.kt b/vripper-core/src/main/kotlin/me/vripper/utilities/Tasks.kt index 659de3a..ce5d9bc 100644 --- a/vripper-core/src/main/kotlin/me/vripper/utilities/Tasks.kt +++ b/vripper-core/src/main/kotlin/me/vripper/utilities/Tasks.kt @@ -1,5 +1,9 @@ package me.vripper.utilities +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch import me.vripper.event.EventBus import me.vripper.event.LoadingTasks import org.koin.core.component.KoinComponent @@ -9,11 +13,14 @@ object Tasks : KoinComponent { private val eventBus: EventBus by inject() private var current = 0 + private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default) @Synchronized fun increment() { if (current == 0) { - eventBus.publishEvent(LoadingTasks(true)) + coroutineScope.launch { + eventBus.publishEvent(LoadingTasks(true)) + } } current += 1 } @@ -22,7 +29,9 @@ object Tasks : KoinComponent { fun decrement() { current -= 1 if (current == 0) { - eventBus.publishEvent(LoadingTasks(false)) + coroutineScope.launch { + eventBus.publishEvent(LoadingTasks(false)) + } } } } diff --git a/vripper-core/src/main/resources/db.changelog-master.xml b/vripper-core/src/main/resources/db.changelog-master.xml index 36fcdda..c937464 100644 --- a/vripper-core/src/main/resources/db.changelog-master.xml +++ b/vripper-core/src/main/resources/db.changelog-master.xml @@ -145,9 +145,24 @@ - - - - - + + + + + + + + + + + + + + + + + + + + diff --git a/vripper-gui/pom.xml b/vripper-gui/pom.xml index 56baf2c..304ba60 100644 --- a/vripper-gui/pom.xml +++ b/vripper-gui/pom.xml @@ -29,6 +29,10 @@ kotlinx-coroutines-core-jvm + org.jetbrains.kotlinx + kotlinx-serialization-json + + javafx-base org.openjfx ${javafx.version} @@ -119,19 +123,21 @@ compile - - - src/main/kotlin - target/generated-sources/annotations - - - - -Xjsr305=strict - + 1.8 + + kotlinx-serialization + + + + org.jetbrains.kotlin + kotlin-maven-serialization + ${kotlin.version} + + maven-jar-plugin diff --git a/vripper-gui/src/main/kotlin/me/vripper/gui/clipboard/ClipboardService.kt b/vripper-gui/src/main/kotlin/me/vripper/gui/clipboard/ClipboardService.kt index d201845..888520f 100644 --- a/vripper-gui/src/main/kotlin/me/vripper/gui/clipboard/ClipboardService.kt +++ b/vripper-gui/src/main/kotlin/me/vripper/gui/clipboard/ClipboardService.kt @@ -2,12 +2,13 @@ package me.vripper.gui.clipboard import javafx.scene.input.Clipboard import kotlinx.coroutines.* +import kotlinx.coroutines.flow.filterIsInstance import me.vripper.event.EventBus import me.vripper.event.SettingsUpdateEvent import me.vripper.model.Settings import me.vripper.services.AppEndpointService import me.vripper.services.SettingsService -import tornadofx.runLater +import tornadofx.* class ClipboardService( private val appEndpointService: AppEndpointService, @@ -19,8 +20,10 @@ class ClipboardService( private var pollJob: Job? = null fun init() { - eventBus.events.ofType(SettingsUpdateEvent::class.java).subscribe { - run(it.settings) + coroutineScope.launch { + eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collect { + run(it.settings) + } } run(settingsService.settings) } diff --git a/vripper-gui/src/main/kotlin/me/vripper/gui/controller/GlobalStateController.kt b/vripper-gui/src/main/kotlin/me/vripper/gui/controller/GlobalStateController.kt index c4e8910..6ae288b 100644 --- a/vripper-gui/src/main/kotlin/me/vripper/gui/controller/GlobalStateController.kt +++ b/vripper-gui/src/main/kotlin/me/vripper/gui/controller/GlobalStateController.kt @@ -1,11 +1,14 @@ package me.vripper.gui.controller +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.filterIsInstance +import kotlinx.coroutines.time.sample import me.vripper.event.* +import me.vripper.event.EventBus import me.vripper.gui.model.GlobalStateModel import me.vripper.services.VGAuthService import me.vripper.utilities.formatSI -import tornadofx.Controller -import tornadofx.runLater +import tornadofx.* import java.time.Duration class GlobalStateController : Controller() { @@ -13,40 +16,51 @@ class GlobalStateController : Controller() { private val eventBus: EventBus by di() private val vgAuthService: VGAuthService by di() - var globalState: GlobalStateModel = GlobalStateModel(0, 0, 0, vgAuthService.loggedUser, 0L.formatSI(), false) + var globalState: GlobalStateModel = + GlobalStateModel(0, 0, 0, vgAuthService.loggedUser, 0L.formatSI(), false) + private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + @OptIn(FlowPreview::class) fun init() { - eventBus.events.ofType(DownloadSpeedEvent::class.java).subscribe { - runLater { - globalState.downloadSpeed = it.downloadSpeed.speed.formatSI() - } - } - - eventBus.events.ofType(VGUserLoginEvent::class.java).subscribe { - runLater { - globalState.loggedUser = it.username - } - } - - eventBus.events.ofType(QueueStateEvent::class.java).subscribe { - runLater { - globalState.apply { - running = it.queueState.running - remaining = it.queueState.remaining + coroutineScope.launch { + eventBus.events.filterIsInstance(DownloadSpeedEvent::class).collect { + runLater { + globalState.downloadSpeed = it.downloadSpeed.speed.formatSI() } } } - eventBus.events.ofType(ErrorCountEvent::class.java).subscribe { - runLater { - globalState.error = it.errorCount.count + coroutineScope.launch { + eventBus.events.filterIsInstance(VGUserLoginEvent::class).collect { + runLater { + globalState.loggedUser = it.username + } } } - - eventBus.events.ofType(LoadingTasks::class.java).sample(Duration.ofMillis(500)).subscribe { - runLater { - globalState.loading = it.loading + coroutineScope.launch { + eventBus.events.filterIsInstance(QueueStateEvent::class).collect { + runLater { + globalState.apply { + running = it.queueState.running + remaining = it.queueState.remaining + } + } } } + coroutineScope.launch { + eventBus.events.filterIsInstance(ErrorCountEvent::class).collect { + runLater { + globalState.error = it.errorCount.count + } + } + } + coroutineScope.launch { + eventBus.events.filterIsInstance(LoadingTasks::class).sample(Duration.ofMillis(500)) + .collect { + runLater { + globalState.loading = it.loading + } + } + } } } \ No newline at end of file diff --git a/vripper-gui/src/main/kotlin/me/vripper/gui/controller/ImageController.kt b/vripper-gui/src/main/kotlin/me/vripper/gui/controller/ImageController.kt index c5e43b0..8694058 100644 --- a/vripper-gui/src/main/kotlin/me/vripper/gui/controller/ImageController.kt +++ b/vripper-gui/src/main/kotlin/me/vripper/gui/controller/ImageController.kt @@ -3,7 +3,7 @@ package me.vripper.gui.controller import me.vripper.entities.Image import me.vripper.gui.model.ImageModel import me.vripper.services.DataTransaction -import tornadofx.Controller +import tornadofx.* class ImageController : Controller() { @@ -21,7 +21,8 @@ class ImageController : Controller() { progress(it.size, it.downloaded), it.status.name, it.size, - it.downloaded + it.downloaded, + it.filename ) } diff --git a/vripper-gui/src/main/kotlin/me/vripper/gui/controller/PostController.kt b/vripper-gui/src/main/kotlin/me/vripper/gui/controller/PostController.kt index 873c7ef..7e98199 100644 --- a/vripper-gui/src/main/kotlin/me/vripper/gui/controller/PostController.kt +++ b/vripper-gui/src/main/kotlin/me/vripper/gui/controller/PostController.kt @@ -2,11 +2,12 @@ package me.vripper.gui.controller import kotlinx.coroutines.* import me.vripper.entities.Image +import me.vripper.entities.Metadata import me.vripper.gui.model.PostModel import me.vripper.services.AppEndpointService import me.vripper.services.DataTransaction import me.vripper.utilities.formatSI -import tornadofx.Controller +import tornadofx.* import java.time.format.DateTimeFormatter class PostController : Controller() { @@ -62,6 +63,7 @@ class PostController : Controller() { fun mapper(id: Long): PostModel { val post = dataTransaction.findPostById(id).orElseThrow() + val metadata = dataTransaction.findMetadataByPostId(post.postId) return PostModel( post.postId, post.postTitle, @@ -76,7 +78,8 @@ class PostController : Controller() { post.getDownloadFolder(), post.folderName, progressCount(post.total, post.done, post.downloaded), - dataTransaction.findImagesByPostId(post.postId).map(Image::thumbUrl).take(4) + dataTransaction.findImagesByPostId(post.postId).map(Image::thumbUrl).take(4), + metadata.orElse(Metadata(post.postId, Metadata.Data("", emptyList()))) ) } diff --git a/vripper-gui/src/main/kotlin/me/vripper/gui/controller/SettingsController.kt b/vripper-gui/src/main/kotlin/me/vripper/gui/controller/SettingsController.kt index 4677ac4..5b2af1e 100644 --- a/vripper-gui/src/main/kotlin/me/vripper/gui/controller/SettingsController.kt +++ b/vripper-gui/src/main/kotlin/me/vripper/gui/controller/SettingsController.kt @@ -6,7 +6,7 @@ import me.vripper.gui.model.settings.SystemSettingsModel import me.vripper.gui.model.settings.ViperSettingsModel import me.vripper.model.* import me.vripper.services.SettingsService -import tornadofx.Controller +import tornadofx.* class SettingsController : Controller() { @@ -34,31 +34,32 @@ class SettingsController : Controller() { viperSettingsModel: ViperSettingsModel, systemSettingsModel: SystemSettingsModel ) { - settingsService.newSettings(Settings().apply { - downloadSettings = DownloadSettings( - downloadSettingsModel.downloadPath, - downloadSettingsModel.autoStart, - downloadSettingsModel.autoQueueThreshold, - downloadSettingsModel.forceOrder, - downloadSettingsModel.forumSubfolder, - downloadSettingsModel.threadSubLocation, - downloadSettingsModel.clearCompleted, - downloadSettingsModel.appendPostId - ) - connectionSettings = ConnectionSettings( - connectionSettingsModel.maxThreads, - connectionSettingsModel.maxTotalThreads, - connectionSettingsModel.timeout, - connectionSettingsModel.maxAttempts, - ) - viperSettings = ViperSettings( - viperSettingsModel.login, - viperSettingsModel.username, - viperSettingsModel.password, - viperSettingsModel.thanks, - viperSettingsModel.host, - ) - systemSettings = + settingsService.newSettings( + Settings( + downloadSettings = DownloadSettings( + downloadSettingsModel.downloadPath, + downloadSettingsModel.autoStart, + downloadSettingsModel.autoQueueThreshold, + downloadSettingsModel.forceOrder, + downloadSettingsModel.forumSubfolder, + downloadSettingsModel.threadSubLocation, + downloadSettingsModel.clearCompleted, + downloadSettingsModel.appendPostId + ), + connectionSettings = ConnectionSettings( + connectionSettingsModel.maxThreads, + connectionSettingsModel.maxTotalThreads, + connectionSettingsModel.timeout, + connectionSettingsModel.maxAttempts, + ), + viperSettings = ViperSettings( + viperSettingsModel.login, + viperSettingsModel.username, + viperSettingsModel.password, + viperSettingsModel.thanks, + viperSettingsModel.host, + ), + systemSettings = SystemSettings( systemSettingsModel.tempPath, systemSettingsModel.cachePath, @@ -66,8 +67,8 @@ class SettingsController : Controller() { if (systemSettingsModel.pollingRate.isBlank()) 500 else systemSettingsModel.pollingRate.toInt(), systemSettingsModel.logEntries.toInt() ) - }) - + ) + ) } fun getProxies(): List { diff --git a/vripper-gui/src/main/kotlin/me/vripper/gui/model/ImageModel.kt b/vripper-gui/src/main/kotlin/me/vripper/gui/model/ImageModel.kt index 1ad58ec..964d112 100644 --- a/vripper-gui/src/main/kotlin/me/vripper/gui/model/ImageModel.kt +++ b/vripper-gui/src/main/kotlin/me/vripper/gui/model/ImageModel.kt @@ -5,8 +5,7 @@ import javafx.beans.property.SimpleIntegerProperty import javafx.beans.property.SimpleLongProperty import javafx.beans.property.SimpleStringProperty import me.vripper.utilities.formatSI -import tornadofx.getValue -import tornadofx.setValue +import tornadofx.* class ImageModel( id: Long, @@ -15,7 +14,8 @@ class ImageModel( progress: Double, status: String, size: Long, - downloaded: Long + downloaded: Long, + filename: String ) { val idProperty = SimpleLongProperty(id) @@ -33,6 +33,9 @@ class ImageModel( val statusProperty = SimpleStringProperty(status) var status: String by statusProperty + val filenameProperty = SimpleStringProperty(filename) + var filename: String by filenameProperty + val sizeProperty = SimpleStringProperty(size.formatSI()) var size = size set(value) { diff --git a/vripper-gui/src/main/kotlin/me/vripper/gui/model/PostModel.kt b/vripper-gui/src/main/kotlin/me/vripper/gui/model/PostModel.kt index f77f974..0a1b40f 100644 --- a/vripper-gui/src/main/kotlin/me/vripper/gui/model/PostModel.kt +++ b/vripper-gui/src/main/kotlin/me/vripper/gui/model/PostModel.kt @@ -1,11 +1,10 @@ package me.vripper.gui.model -import javafx.beans.property.SimpleDoubleProperty -import javafx.beans.property.SimpleIntegerProperty -import javafx.beans.property.SimpleLongProperty -import javafx.beans.property.SimpleStringProperty -import tornadofx.getValue -import tornadofx.setValue +import javafx.beans.property.* +import javafx.collections.FXCollections +import javafx.collections.ObservableList +import me.vripper.entities.Metadata +import tornadofx.* class PostModel( postId: Long, @@ -21,7 +20,8 @@ class PostModel( path: String, folderName: String, progressCount: String, - previewList: List + previewList: List, + metadata: Metadata ) { val postIdProperty = SimpleLongProperty(postId) var postId: Long by postIdProperty @@ -64,4 +64,11 @@ class PostModel( val previewListProperty = SimpleStringProperty(previewList.joinToString("|")) var previewList: List = previewListProperty.value.split("|") + + val altTitlesProperty = + SimpleListProperty(FXCollections.observableArrayList(metadata.data.resolvedNames)) + var altTitles: ObservableList by altTitlesProperty + + val postedByProperty = SimpleStringProperty(metadata.data.postedBy) + var postedby: String by postedByProperty } diff --git a/vripper-gui/src/main/kotlin/me/vripper/gui/view/popup/RenameView.kt b/vripper-gui/src/main/kotlin/me/vripper/gui/view/popup/RenameView.kt index f1b7265..88c4f3b 100644 --- a/vripper-gui/src/main/kotlin/me/vripper/gui/view/popup/RenameView.kt +++ b/vripper-gui/src/main/kotlin/me/vripper/gui/view/popup/RenameView.kt @@ -2,9 +2,7 @@ package me.vripper.gui.view.popup import javafx.beans.property.SimpleStringProperty import javafx.geometry.Pos -import javafx.scene.control.TextField -import javafx.scene.layout.Priority -import javafx.scene.layout.VBox +import javafx.scene.control.ComboBox import me.vripper.gui.controller.PostController import tornadofx.* @@ -12,29 +10,32 @@ class RenameView : Fragment("Rename download post") { val postId: Long by param() val name: String by param() - private val textAreaProperty = SimpleStringProperty() + val altTitles: List by param() + private val textInputProperty = SimpleStringProperty() private val postController: PostController by inject() - lateinit var input: TextField + private lateinit var comboBox: ComboBox override fun onDock() { - textAreaProperty.value = name + textInputProperty.value = name } override val root = vbox(alignment = Pos.CENTER_RIGHT) { padding = insets(all = 5) spacing = 5.0 - input = textfield { - VBox.setVgrow(this, Priority.ALWAYS) - bind(textAreaProperty) - } - button("Ok") { - imageview("search.png") { - fitWidth = 18.0 - fitHeight = 18.0 + form { + fieldset { + field("Name") { + comboBox = combobox(textInputProperty, altTitles.ifEmpty { listOf(name) }) { + useMaxSize = true + isEditable = true + } + } } - disableWhen(textAreaProperty.isEmpty) + } + button("Rename") { + disableWhen(comboBox.editor.textProperty().isEmpty) action { - postController.rename(postId, textAreaProperty.value) + postController.rename(postId, comboBox.editor.text.trim()) close() } } diff --git a/vripper-gui/src/main/kotlin/me/vripper/gui/view/settings/ConnectionSettingsView.kt b/vripper-gui/src/main/kotlin/me/vripper/gui/view/settings/ConnectionSettingsView.kt index 9852deb..471c2b9 100644 --- a/vripper-gui/src/main/kotlin/me/vripper/gui/view/settings/ConnectionSettingsView.kt +++ b/vripper-gui/src/main/kotlin/me/vripper/gui/view/settings/ConnectionSettingsView.kt @@ -4,7 +4,7 @@ import me.vripper.gui.controller.SettingsController import me.vripper.gui.model.settings.ConnectionSettingsModel import tornadofx.* -class ConnectionSettingsView : View("Connection settings") { +class ConnectionSettingsView : View("Connection Settings") { private val settingsController: SettingsController by inject() val connectionSettingsModel = ConnectionSettingsModel() diff --git a/vripper-gui/src/main/kotlin/me/vripper/gui/view/tables/ImagesTableView.kt b/vripper-gui/src/main/kotlin/me/vripper/gui/view/tables/ImagesTableView.kt index d34f175..2b01d8f 100644 --- a/vripper-gui/src/main/kotlin/me/vripper/gui/view/tables/ImagesTableView.kt +++ b/vripper-gui/src/main/kotlin/me/vripper/gui/view/tables/ImagesTableView.kt @@ -9,6 +9,9 @@ import javafx.scene.image.ImageView import javafx.scene.input.MouseButton import javafx.util.Callback import kotlinx.coroutines.* +import kotlinx.coroutines.flow.filter +import kotlinx.coroutines.flow.filterIsInstance +import kotlinx.coroutines.flow.map import me.vripper.entities.Image import me.vripper.event.EventBus import me.vripper.event.ImageEvent @@ -17,14 +20,11 @@ import me.vripper.gui.model.ImageModel import me.vripper.gui.view.ProgressTableCell import me.vripper.gui.view.StatusTableCell import me.vripper.gui.view.openLink -import reactor.core.Disposable import tornadofx.* -import java.time.Duration class ImagesTableView : Fragment("Photos") { private lateinit var tableView: TableView - private lateinit var eventsSubscription: Disposable private val imageController: ImageController by inject() private val eventBus: EventBus by di() private var items: ObservableList = FXCollections.observableArrayList() @@ -47,22 +47,24 @@ class ImagesTableView : Fragment("Photos") { } } - eventsSubscription = eventBus.events.ofType(ImageEvent::class.java).map { - it.copy(images = it.images.filter { image: Image -> image.postId == postId }) - }.filter { - it.images.isNotEmpty() - }.buffer(Duration.ofMillis(175)) - .subscribe { imageEvent -> - runLater { - for (image in imageEvent.map { it.images }.flatten().reversed().distinct()) { - val imageModel = items.find { it.id == image.id } ?: continue + coroutineScope.launch { + eventBus.events.filterIsInstance(ImageEvent::class).map { + it.copy(images = it.images.filter { image: Image -> image.postId == postId }) + }.filter { + it.images.isNotEmpty() + }.collect { imageEvent -> + runLater { + for (image in imageEvent.images) { + val imageModel = items.find { it.id == image.id } ?: continue - imageModel.size = image.size - imageModel.status = image.status.name - imageModel.downloaded = image.downloaded - imageModel.progress = imageController.progress( - image.size, image.downloaded - ) + imageModel.size = image.size + imageModel.status = image.status.name + imageModel.filename = image.filename + imageModel.downloaded = image.downloaded + imageModel.progress = imageController.progress( + image.size, image.downloaded + ) + } } } } @@ -83,8 +85,8 @@ class ImagesTableView : Fragment("Photos") { } val contextMenu = ContextMenu() contextMenu.items.addAll(urlItem) - tableRow.contextMenuProperty() - .bind(tableRow.emptyProperty().map { empty -> if (empty) null else contextMenu }) + tableRow.contextMenuProperty().bind(tableRow.emptyProperty() + .map { empty -> if (empty) null else contextMenu }) tableRow } column("Index", ImageModel::indexProperty) { @@ -120,6 +122,12 @@ class ImagesTableView : Fragment("Photos") { cell as TableCell } } + column("Filename", ImageModel::filenameProperty) { + prefWidth = 150.0 + cellFactory = Callback { + TextFieldTableCell().apply { alignment = Pos.CENTER_LEFT } + } + } column("Status", ImageModel::statusProperty) { prefWidth = 50.0 cellFactory = Callback { @@ -143,6 +151,5 @@ class ImagesTableView : Fragment("Photos") { override fun onUndock() { coroutineScope.cancel() - eventsSubscription.dispose() } } \ No newline at end of file diff --git a/vripper-gui/src/main/kotlin/me/vripper/gui/view/tables/LogTableView.kt b/vripper-gui/src/main/kotlin/me/vripper/gui/view/tables/LogTableView.kt index 2da89b9..a5187b0 100644 --- a/vripper-gui/src/main/kotlin/me/vripper/gui/view/tables/LogTableView.kt +++ b/vripper-gui/src/main/kotlin/me/vripper/gui/view/tables/LogTableView.kt @@ -2,6 +2,7 @@ package me.vripper.gui.view.tables import javafx.scene.control.* import kotlinx.coroutines.* +import kotlinx.coroutines.flow.filterIsInstance import me.vripper.event.EventBus import me.vripper.event.LogCreateEvent import me.vripper.event.LogDeleteEvent @@ -41,28 +42,34 @@ class LogTableView : View() { } } - eventBus.events.ofType(LogCreateEvent::class.java).subscribe { - val logModel = logController.mapper(it.logEntry) - runLater { - items.add(logModel) + coroutineScope.launch { + eventBus.events.filterIsInstance(LogCreateEvent::class).collect { + val logModel = logController.mapper(it.logEntry) + runLater { + items.add(logModel) + } } } - - eventBus.events.ofType(LogUpdateEvent::class.java).subscribe { - val logModel = logController.mapper(it.logEntry) - val find = items.find { threadModel -> threadModel.id == logModel.id } - if (find != null) { - runLater { - find.apply { - status = logModel.status - message = logModel.message + coroutineScope.launch { + eventBus.events.filterIsInstance(LogUpdateEvent::class).collect { + val logModel = logController.mapper(it.logEntry) + val find = items.find { threadModel -> threadModel.id == logModel.id } + if (find != null) { + runLater { + find.apply { + status = logModel.status + message = logModel.message + } } } } } - eventBus.events.ofType(LogDeleteEvent::class.java).subscribe { deleteEvent -> - runLater { - items.items.removeIf { it.id in deleteEvent.deleted } + + coroutineScope.launch { + eventBus.events.filterIsInstance(LogDeleteEvent::class).collect { deleteEvent -> + runLater { + items.items.removeIf { it.id in deleteEvent.deleted } + } } } } diff --git a/vripper-gui/src/main/kotlin/me/vripper/gui/view/tables/PostsTableView.kt b/vripper-gui/src/main/kotlin/me/vripper/gui/view/tables/PostsTableView.kt index fd4ea8f..acd7cdf 100644 --- a/vripper-gui/src/main/kotlin/me/vripper/gui/view/tables/PostsTableView.kt +++ b/vripper-gui/src/main/kotlin/me/vripper/gui/view/tables/PostsTableView.kt @@ -12,11 +12,10 @@ import javafx.util.Callback import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.filterIsInstance import kotlinx.coroutines.launch +import me.vripper.event.* import me.vripper.event.EventBus -import me.vripper.event.PostCreateEvent -import me.vripper.event.PostDeleteEvent -import me.vripper.event.PostUpdateEvent import me.vripper.gui.clipboard.ClipboardService import me.vripper.gui.controller.PostController import me.vripper.gui.model.PostModel @@ -57,40 +56,60 @@ class PostsTableView : View() { } } - eventBus.events.ofType(PostCreateEvent::class.java).subscribe { postEvents -> - val add = postEvents.posts.map { postController.mapper(it.id) } - runLater { - tableView.items.addAll(add) - } - } - - eventBus.events.ofType(PostUpdateEvent::class.java).subscribe { postEvents -> - runLater { - for (post in postEvents.posts) { - val postModel = items.find { it.postId == post.postId } ?: continue - - postModel.status = post.status.name - postModel.progressCount = postController.progressCount( - post.total, post.done, post.downloaded - ) - postModel.order = post.rank + 1 - postModel.done = post.done - postModel.progress = postController.progress( - post.total, post.done - ) - postModel.path = post.getDownloadFolder() - postModel.folderName = post.folderName + coroutineScope.launch { + eventBus.events.filterIsInstance(PostCreateEvent::class).collect { postEvents -> + val add = postEvents.posts.map { postController.mapper(it.id) } + runLater { + tableView.items.addAll(add) } } } - eventBus.events.ofType(PostDeleteEvent::class.java).subscribe { postEvents -> - runLater { - postEvents.postIds.forEach { - items.removeIf { p -> p.postId == it } + coroutineScope.launch { + eventBus.events.filterIsInstance(PostUpdateEvent::class).collect { postEvents -> + runLater { + for (post in postEvents.posts) { + val postModel = items.find { it.postId == post.postId } ?: continue + + postModel.status = post.status.name + postModel.progressCount = postController.progressCount( + post.total, post.done, post.downloaded + ) + postModel.order = post.rank + 1 + postModel.done = post.done + postModel.progress = postController.progress( + post.total, post.done + ) + postModel.path = post.getDownloadFolder() + postModel.folderName = post.folderName + } } } } + + coroutineScope.launch { + eventBus.events.filterIsInstance(PostDeleteEvent::class).collect { postEvents -> + runLater { + postEvents.postIds.forEach { + items.removeIf { p -> p.postId == it } + } + } + } + } + coroutineScope.launch { + eventBus.events.filterIsInstance(MetadataUpdateEvent::class) + .collect { metadataUpdateEvent -> + runLater { + val postModel = + items.find { it.postId == metadataUpdateEvent.metadata.postId } + ?: return@runLater + + postModel.altTitles = + FXCollections.observableArrayList(metadataUpdateEvent.metadata.data.resolvedNames) + postModel.postedby = metadataUpdateEvent.metadata.data.postedBy + } + } + } } override val root = vbox { @@ -176,10 +195,17 @@ class PostsTableView : View() { val contextMenu = ContextMenu() contextMenu.items.addAll( - startItem, stopItem, renameItem, deleteItem, SeparatorMenuItem(), detailsItem, locationItem, urlItem + startItem, + stopItem, + renameItem, + deleteItem, + SeparatorMenuItem(), + detailsItem, + locationItem, + urlItem ) - tableRow.contextMenuProperty() - .bind(tableRow.emptyProperty().map { empty -> if (empty) null else contextMenu }) + tableRow.contextMenuProperty().bind(tableRow.emptyProperty() + .map { empty -> if (empty) null else contextMenu }) tableRow } column("Preview", PostModel::previewListProperty) { @@ -287,6 +313,12 @@ class PostsTableView : View() { TextFieldTableCell().apply { alignment = Pos.CENTER_LEFT } } } + column("Posted By", PostModel::postedByProperty) { + prefWidth = 100.0 + cellFactory = Callback { + TextFieldTableCell().apply { alignment = Pos.CENTER_LEFT } + } + } column("Order", PostModel::orderProperty) { prefWidth = 50.0 sortOrder.add(this) @@ -298,10 +330,15 @@ class PostsTableView : View() { } private fun rename(post: PostModel) { - find(mapOf(RenameView::postId to post.postId, RenameView::name to post.folderName)).openModal() - ?.apply { - minWidth = 300.0 - } + find( + mapOf( + RenameView::postId to post.postId, + RenameView::name to post.folderName, + RenameView::altTitles to post.altTitles + ) + ).openModal()?.apply { + minWidth = 450.0 + } } fun renameSelected() { @@ -336,8 +373,8 @@ class PostsTableView : View() { private fun openPhotos(postId: Long) { find(mapOf(ImagesTableView::postId to postId)).openModal()?.apply { - minWidth = 600.0 - minHeight = 400.0 + minWidth = 800.0 + minHeight = 600.0 } } } \ No newline at end of file diff --git a/vripper-gui/src/main/kotlin/me/vripper/gui/view/tables/ThreadTableView.kt b/vripper-gui/src/main/kotlin/me/vripper/gui/view/tables/ThreadTableView.kt index 79d6534..31be60f 100644 --- a/vripper-gui/src/main/kotlin/me/vripper/gui/view/tables/ThreadTableView.kt +++ b/vripper-gui/src/main/kotlin/me/vripper/gui/view/tables/ThreadTableView.kt @@ -5,6 +5,7 @@ import javafx.collections.ObservableList import javafx.scene.control.* import javafx.scene.image.ImageView import kotlinx.coroutines.* +import kotlinx.coroutines.flow.filterIsInstance import me.vripper.event.EventBus import me.vripper.event.ThreadClearEvent import me.vripper.event.ThreadCreateEvent @@ -44,22 +45,28 @@ class ThreadTableView : View() { tableView.placeholder = Label("No content in table") } } - eventBus.events.ofType(ThreadCreateEvent::class.java).subscribe { - val threadModelMapper = threadController.threadModelMapper(it.thread) - runLater { - items.add(threadModelMapper) + coroutineScope.launch { + eventBus.events.filterIsInstance(ThreadCreateEvent::class).collect { + val threadModelMapper = threadController.threadModelMapper(it.thread) + runLater { + items.add(threadModelMapper) + } } } - eventBus.events.ofType(ThreadDeleteEvent::class.java).subscribe { event -> - runLater { - tableView.items.removeIf { it.threadId == event.threadId } + coroutineScope.launch { + eventBus.events.filterIsInstance(ThreadDeleteEvent::class).collect { event -> + runLater { + tableView.items.removeIf { it.threadId == event.threadId } + } } } - eventBus.events.ofType(ThreadClearEvent::class.java).subscribe { - runLater { - tableView.items.clear() + coroutineScope.launch { + eventBus.events.filterIsInstance(ThreadClearEvent::class).collect { + runLater { + tableView.items.clear() + } } } } @@ -108,8 +115,8 @@ class ThreadTableView : View() { val contextMenu = ContextMenu() contextMenu.items.addAll(selectItem, urlItem, SeparatorMenuItem(), deleteItem) - tableRow.contextMenuProperty() - .bind(tableRow.emptyProperty().map { empty -> if (empty) null else contextMenu }) + tableRow.contextMenuProperty().bind(tableRow.emptyProperty() + .map { empty -> if (empty) null else contextMenu }) tableRow } column("Title", ThreadModel::titleProperty) { @@ -135,11 +142,12 @@ class ThreadTableView : View() { } private fun selectPosts(threadId: Long) { - find(mapOf(ThreadSelectionTableView::threadId to threadId)).openModal()?.apply { - minWidth = 600.0 - minHeight = 400.0 - width = 800.0 - height = 600.0 - } + find(mapOf(ThreadSelectionTableView::threadId to threadId)).openModal() + ?.apply { + minWidth = 600.0 + minHeight = 400.0 + width = 800.0 + height = 600.0 + } } } \ No newline at end of file diff --git a/vripper-web/src/main/kotlin/me/vripper/web/wsendpoints/DataBroadcast.kt b/vripper-web/src/main/kotlin/me/vripper/web/wsendpoints/DataBroadcast.kt index c6178f8..53c8805 100644 --- a/vripper-web/src/main/kotlin/me/vripper/web/wsendpoints/DataBroadcast.kt +++ b/vripper-web/src/main/kotlin/me/vripper/web/wsendpoints/DataBroadcast.kt @@ -1,6 +1,10 @@ package me.vripper.web.wsendpoints import jakarta.annotation.PostConstruct +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.filterIsInstance +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.time.sample import me.vripper.event.* import org.koin.core.component.KoinComponent import org.koin.core.component.inject @@ -14,72 +18,110 @@ class DataBroadcast( ) : KoinComponent { private val eventBus: EventBus by inject() + private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default) + @OptIn(FlowPreview::class) @PostConstruct private fun run() { - eventBus.events.ofType(PostCreateEvent::class.java).subscribe { events -> - template.convertAndSend("/topic/posts/new", events.posts) - } - eventBus.events.ofType(PostUpdateEvent::class.java).subscribe { events -> - template.convertAndSend("/topic/posts/updated", events.posts) - } - eventBus.events.ofType(PostDeleteEvent::class.java).subscribe { events -> - template.convertAndSend("/topic/posts/deleted", events.postIds) - } - - eventBus.events.ofType(QueueStateEvent::class.java).subscribe { - template.convertAndSend("/topic/queue-state", it.queueState) - } - - eventBus.events.ofType(DownloadSpeedEvent::class.java).subscribe { - template.convertAndSend("/topic/download-speed", it.downloadSpeed) - } - - eventBus.events.ofType(VGUserLoginEvent::class.java).subscribe { - template.convertAndSend("/topic/vg-username", it.username) - } - - eventBus.events.ofType(ErrorCountEvent::class.java).subscribe { - template.convertAndSend("/topic/error-count", it.errorCount) - } - - eventBus.events.ofType(LoadingTasks::class.java).sample(Duration.ofMillis(500)).subscribe { - template.convertAndSend("/topic/loading", it.loading) - } - - eventBus.events.ofType(ImageEvent::class.java).buffer(Duration.ofMillis(500)).subscribe { events -> - events.reversed().map { it.images }.flatten().distinct().groupBy { it.postId } - .forEach { - template.convertAndSend("/topic/images/${it.key}", it.value) + coroutineScope.launch { + coroutineScope.launch { + eventBus.events.filterIsInstance(PostCreateEvent::class).collect { events -> + template.convertAndSend("/topic/posts/new", events.posts) } - } - - eventBus.events.ofType(ThreadCreateEvent::class.java).map { listOf(it.thread) } - .subscribe { events -> - template.convertAndSend("/topic/threads", events) } - eventBus.events.ofType(ThreadDeleteEvent::class.java).map { listOf(it.threadId) } - .subscribe { - template.convertAndSend("/topic/threads/deleted", it) + coroutineScope.launch { + eventBus.events.filterIsInstance(PostUpdateEvent::class).collect { events -> + template.convertAndSend("/topic/posts/updated", events.posts) + } } - eventBus.events.ofType(ThreadClearEvent::class.java).map { listOf(true) }.subscribe { - template.convertAndSend("/topic/threads/deletedAll", it) - } - - eventBus.events.ofType(LogCreateEvent::class.java).map { it.logEntry } - .subscribe { logCreateEvent -> - template.convertAndSend("/topic/logs/new", listOf(logCreateEvent)) + coroutineScope.launch { + eventBus.events.filterIsInstance(PostDeleteEvent::class).collect { events -> + template.convertAndSend("/topic/posts/deleted", events.postIds) + } } - eventBus.events.ofType(LogUpdateEvent::class.java).map { it.logEntry } - .subscribe { logUpdateEvent -> - template.convertAndSend("/topic/logs/updated", listOf(logUpdateEvent)) + coroutineScope.launch { + eventBus.events.filterIsInstance(QueueStateEvent::class).collect { + template.convertAndSend("/topic/queue-state", it.queueState) + } } - eventBus.events.ofType(LogDeleteEvent::class.java).subscribe { - template.convertAndSend("/topic/logs/deleted", listOf(it.deleted)) + coroutineScope.launch { + eventBus.events.filterIsInstance(DownloadSpeedEvent::class).collect { + template.convertAndSend("/topic/download-speed", it.downloadSpeed) + } + } + + coroutineScope.launch { + eventBus.events.filterIsInstance(VGUserLoginEvent::class).collect { + template.convertAndSend("/topic/vg-username", it.username) + } + } + + coroutineScope.launch { + eventBus.events.filterIsInstance(ErrorCountEvent::class).collect { + template.convertAndSend("/topic/error-count", it.errorCount) + } + } + + coroutineScope.launch { + eventBus.events.filterIsInstance(LoadingTasks::class).sample(Duration.ofMillis(500)) + .collect { + template.convertAndSend("/topic/loading", it.loading) + } + } + + coroutineScope.launch { + eventBus.events.filterIsInstance(ImageEvent::class).collect { events -> + events.images.groupBy { it.postId }.forEach { + template.convertAndSend("/topic/images/${it.key}", it.value) + } + } + } + + coroutineScope.launch { + eventBus.events.filterIsInstance(ThreadCreateEvent::class).map { listOf(it.thread) } + .collect { events -> + template.convertAndSend("/topic/threads", events) + } + } + + coroutineScope.launch { + eventBus.events.filterIsInstance(ThreadDeleteEvent::class) + .map { listOf(it.threadId) } + .collect { + template.convertAndSend("/topic/threads/deleted", it) + } + } + + coroutineScope.launch { + eventBus.events.filterIsInstance(ThreadClearEvent::class).map { listOf(true) } + .collect { + template.convertAndSend("/topic/threads/deletedAll", it) + } + } + + coroutineScope.launch { + eventBus.events.filterIsInstance(LogCreateEvent::class).map { it.logEntry } + .collect { logCreateEvent -> + template.convertAndSend("/topic/logs/new", listOf(logCreateEvent)) + } + } + + coroutineScope.launch { + eventBus.events.filterIsInstance(LogUpdateEvent::class).map { it.logEntry } + .collect { logUpdateEvent -> + template.convertAndSend("/topic/logs/updated", listOf(logUpdateEvent)) + } + } + + coroutineScope.launch { + eventBus.events.filterIsInstance(LogDeleteEvent::class).collect { + template.convertAndSend("/topic/logs/deleted", listOf(it.deleted)) + } + } } } } \ No newline at end of file