diff --git a/vripper-core/src/main/kotlin/me/vripper/Module.kt b/vripper-core/src/main/kotlin/me/vripper/Module.kt index 05b6281..4d88dcc 100644 --- a/vripper-core/src/main/kotlin/me/vripper/Module.kt +++ b/vripper-core/src/main/kotlin/me/vripper/Module.kt @@ -76,7 +76,7 @@ val coreModule = module { MetadataService(get(), get()) } single { - AcidimgHost(get()) + AcidimgHost() } bind Host::class single { DPicMeHost() diff --git a/vripper-core/src/main/kotlin/me/vripper/host/AcidimgHost.kt b/vripper-core/src/main/kotlin/me/vripper/host/AcidimgHost.kt index 53867d2..7b92854 100644 --- a/vripper-core/src/main/kotlin/me/vripper/host/AcidimgHost.kt +++ b/vripper-core/src/main/kotlin/me/vripper/host/AcidimgHost.kt @@ -2,7 +2,6 @@ package me.vripper.host import me.vripper.exception.HostException import me.vripper.exception.XpathException -import me.vripper.services.HTTPService import me.vripper.services.download.ImageDownloadRunnable.Context import me.vripper.utilities.HtmlUtils import me.vripper.utilities.LoggerDelegate @@ -12,9 +11,7 @@ import org.apache.hc.client5.http.entity.UrlEncodedFormEntity import org.apache.hc.core5.http.message.BasicNameValuePair import org.w3c.dom.Node -internal class AcidimgHost( - private val httpService: HTTPService, -) : Host("acidimg.cc", 0) { +internal class AcidimgHost : Host("acidimg.cc", 0) { private val log by LoggerDelegate() @Throws(HostException::class) diff --git a/vripper-core/src/main/kotlin/me/vripper/host/Host.kt b/vripper-core/src/main/kotlin/me/vripper/host/Host.kt index a8d3e66..9fab3a0 100644 --- a/vripper-core/src/main/kotlin/me/vripper/host/Host.kt +++ b/vripper-core/src/main/kotlin/me/vripper/host/Host.kt @@ -31,7 +31,7 @@ internal abstract class Host( ) : KoinComponent { private val log by LoggerDelegate() - private val httpService: HTTPService by inject() + protected val httpService: HTTPService by inject() private val dataAccessService: DataAccessService by inject() private val downloadSpeedService: DownloadSpeedService by inject() @@ -61,7 +61,7 @@ internal abstract class Host( val imageMimeType = getImageMimeType(headers) val downloadedImage = if (imageMimeType != null) { // a direct link, awesome - val downloadedImage = fetch(context.imageEntity.url, context.imageEntity.url, context) { + val downloadedImage = fetch(context.imageEntity.url, context) { handleImageDownload(it, context) } DownloadedImage(getDefaultImageName(context.imageEntity.url), downloadedImage.first, downloadedImage.second) @@ -84,7 +84,7 @@ internal abstract class Host( private fun downloadByHost(context: Context): DownloadedImage { val resolvedImage = resolve(context) val downloadImage: Pair = - fetch(resolvedImage.second, context.imageEntity.url, context) { + fetch(resolvedImage.second, context) { handleImageDownload(it, context) } return DownloadedImage(resolvedImage.first, downloadImage.first, downloadImage.second) @@ -152,6 +152,7 @@ internal abstract class Host( fun head(context: Context): Array
{ val httpHead = HttpHead(context.imageEntity.url).also { + it.addHeader("Referer", "https://vipergirls.to/") it.setAbsoluteRequestUri(true) context.requests.add(it) } @@ -169,8 +170,8 @@ internal abstract class Host( fun fetch( url: String, - referer: String, context: Context, + referer: String = "https://vipergirls.to/", transformer: (ClassicHttpResponse) -> T ): T { val httpGet = @@ -191,7 +192,7 @@ internal abstract class Host( url: String, context: Context ): Document { - return fetch(url, url, context) { + return fetch(url, context) { HtmlUtils.clean(it.entity.content) }.also { if (log.isDebugEnabled) { diff --git a/vripper-core/src/main/kotlin/me/vripper/host/ImageBamHost.kt b/vripper-core/src/main/kotlin/me/vripper/host/ImageBamHost.kt index 5a23598..1bb0c28 100644 --- a/vripper-core/src/main/kotlin/me/vripper/host/ImageBamHost.kt +++ b/vripper-core/src/main/kotlin/me/vripper/host/ImageBamHost.kt @@ -8,7 +8,6 @@ import me.vripper.utilities.LoggerDelegate import me.vripper.utilities.XpathUtils import org.apache.hc.client5.http.impl.cookie.BasicClientCookie import org.w3c.dom.Node -import java.sql.Date import java.time.LocalDateTime import java.time.ZoneId import java.util.* @@ -24,15 +23,13 @@ internal class ImageBamHost : Host("imagebam.com", 2) { val doc = try { log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_XPATH, context.imageEntity.url)) if (XpathUtils.getAsNode(document, CONTINUE_XPATH) != null) { - val clientCookie = BasicClientCookie("nsfw_inter", "1") - clientCookie.domain = "www.imagebam.com" - clientCookie.path = "/" - clientCookie.expiryDate = - Date.from( - LocalDateTime.now().plusDays(3).atZone(ZoneId.systemDefault()).toInstant() - ) - context.httpContext.cookieStore.addCookie(clientCookie) - fetch(context.imageEntity.url, context.imageEntity.url, context) { + val sfwCookie = BasicClientCookie("sfw_inter", "1") + sfwCookie.domain = "www.imagebam.com" + sfwCookie.path = "/" + sfwCookie.setExpiryDate(LocalDateTime.now().plusDays(3).atZone(ZoneId.systemDefault()).toInstant()) + + context.httpContext.cookieStore.addCookie(sfwCookie) + fetch(context.imageEntity.url, context) { HtmlUtils.clean(it.entity.content) } } else { diff --git a/vripper-core/src/main/kotlin/me/vripper/host/ImageVenueHost.kt b/vripper-core/src/main/kotlin/me/vripper/host/ImageVenueHost.kt index 9ddef3f..7fb106b 100644 --- a/vripper-core/src/main/kotlin/me/vripper/host/ImageVenueHost.kt +++ b/vripper-core/src/main/kotlin/me/vripper/host/ImageVenueHost.kt @@ -26,7 +26,7 @@ internal class ImageVenueHost : Host("imagevenue.com", 4) { ) if (XpathUtils.getAsNode(document, CONTINUE_BUTTON_XPATH) != null) { // Button detected. No need to actually click it, just make the call again. - fetch(context.imageEntity.url, context.imageEntity.url, context) { + fetch(context.imageEntity.url, context) { HtmlUtils.clean(it.entity.content) } } else { diff --git a/vripper-core/src/main/kotlin/me/vripper/host/ImxHost.kt b/vripper-core/src/main/kotlin/me/vripper/host/ImxHost.kt index ed6a1f4..9377b46 100644 --- a/vripper-core/src/main/kotlin/me/vripper/host/ImxHost.kt +++ b/vripper-core/src/main/kotlin/me/vripper/host/ImxHost.kt @@ -3,7 +3,12 @@ package me.vripper.host import me.vripper.entities.ImageEntity import me.vripper.exception.HostException import me.vripper.services.download.ImageDownloadRunnable +import me.vripper.utilities.HtmlUtils import me.vripper.utilities.LoggerDelegate +import me.vripper.utilities.XpathUtils +import org.apache.hc.client5.http.classic.methods.HttpPost +import org.apache.hc.client5.http.entity.UrlEncodedFormEntity +import org.apache.hc.core5.http.message.BasicNameValuePair internal class ImxHost : Host("imx.to", 8) { @@ -15,15 +20,66 @@ internal class ImxHost : Host("imx.to", 8) { context: ImageDownloadRunnable.Context ): Pair { log.debug("Resolving name and image url for ${context.imageEntity.url}") - val imgTitle = String.format("IMG_%04d", context.imageEntity.index + 1) + val imgTitle = getTitle(context).ifEmpty { + getDefaultImageName(context.imageEntity.thumbUrl) + } val imgUrl = findPattern(context.imageEntity) return Pair( - imgTitle.ifEmpty { getDefaultImageName(imgUrl) }, imgUrl + imgTitle, imgUrl ) } - private fun findPattern(image: ImageEntity): String = image.thumbUrl - .replace("http:", "https:") - .replace("upload/small/", "u/i/") - .replace("u/t/", "u/i/") + private fun getTitle(context: ImageDownloadRunnable.Context): String { + return try { + val httpsUrl = context.imageEntity.url.replace("http:", "https:") + val document = fetchDocument(httpsUrl, context) + var value: String? = null + log.debug("Looking for xpath expression $CONTINUE_BUTTON_XPATH in $httpsUrl") + val contDiv = XpathUtils.getAsNode(document, CONTINUE_BUTTON_XPATH) + ?: throw HostException("$CONTINUE_BUTTON_XPATH cannot be found") + val node = contDiv.attributes.getNamedItem("value") + if (node != null) { + value = node.textContent + } + log.debug("Click button found for $httpsUrl") + val httpPost: HttpPost = HttpPost(httpsUrl).also { + it.entity = UrlEncodedFormEntity(listOf(BasicNameValuePair("imgContinue", value))) + }.also { context.requests.add(it) } + log.debug("Requesting {}", httpPost) + val doc = httpService.client.execute( + httpPost, context.httpContext + ) { response -> + log.debug("Cleaning response for {}", httpPost) + HtmlUtils.clean(response.entity.content) + } + + log.debug("Looking for xpath expression $IMG_XPATH in $httpsUrl") + val imgNode = XpathUtils.getAsNode(doc, IMG_XPATH) + + log.debug("Resolving name for $httpsUrl") + val imgTitle = imgNode?.attributes?.getNamedItem("alt")?.textContent?.trim() ?: "" + return imgTitle + } catch (_: Exception) { + "" + } + } + + private fun findPattern(image: ImageEntity): String { + val url = image.thumbUrl + .replace("http:", "https:") + return if (url.startsWith("https://image.imx.to/u/t/")) { + "https://image.imx.to/u/i/" + url.replace("https://image.imx.to/u/t/", "") + } else if (url.startsWith("https://t.imx.to/t/")) { + "https://image.imx.to/u/i/" + url.replace("https://t.imx.to/t/", "") + } else if (url.startsWith("https://imx.to/upload/small/")) { + "https://image.imx.to/u/i/" + url.replace("https://imx.to/upload/small/", "") + } else { + throw HostException("Cannot find pattern for url ${image.thumbUrl}") + } + } + + companion object { + private const val CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']" + private const val IMG_XPATH = "//img[@class='centred']" + } } \ No newline at end of file diff --git a/vripper-core/src/main/kotlin/me/vripper/host/PimpandhostHost.kt b/vripper-core/src/main/kotlin/me/vripper/host/PimpandhostHost.kt index 031aa1f..b8f7524 100644 --- a/vripper-core/src/main/kotlin/me/vripper/host/PimpandhostHost.kt +++ b/vripper-core/src/main/kotlin/me/vripper/host/PimpandhostHost.kt @@ -26,7 +26,7 @@ internal class PimpandhostHost : Host("pimpandhost.com", 9) { } catch (e: Exception) { throw HostException(e) } - val doc = fetch(newUrl, context.imageEntity.url, context) { + val doc = fetch(newUrl, context) { HtmlUtils.clean(it.entity.content) } val imgNode: Node = try { diff --git a/vripper-core/src/main/kotlin/me/vripper/host/PostImgHost.kt b/vripper-core/src/main/kotlin/me/vripper/host/PostImgHost.kt index 4a062ea..fc0d7a0 100644 --- a/vripper-core/src/main/kotlin/me/vripper/host/PostImgHost.kt +++ b/vripper-core/src/main/kotlin/me/vripper/host/PostImgHost.kt @@ -1,12 +1,9 @@ package me.vripper.host import me.vripper.exception.HostException -import me.vripper.exception.XpathException import me.vripper.services.download.ImageDownloadRunnable import me.vripper.utilities.LoggerDelegate import me.vripper.utilities.XpathUtils -import org.w3c.dom.Node -import java.util.* internal class PostImgHost : Host("postimg.cc", 13) { private val log by LoggerDelegate() @@ -15,44 +12,22 @@ internal class PostImgHost : Host("postimg.cc", 13) { override fun resolve( context: ImageDownloadRunnable.Context ): Pair { - val document = fetchDocument(context.imageEntity.url, context) - val titleNode = try { - log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, context.imageEntity.url)) - XpathUtils.getAsNode(document, TITLE_XPATH) - } catch (e: XpathException) { - throw HostException(e) - } ?: throw HostException( - String.format( - "Xpath '%s' cannot be found in '%s'", - TITLE_XPATH, - context.imageEntity.url - ) - ) - val urlNode = try { - log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url)) - XpathUtils.getAsNode(document, IMG_XPATH) - } catch (e: XpathException) { - throw HostException(e) - } ?: throw HostException( + val document = fetchDocument(context.imageEntity.url.replace("http:", "https:"), context) + log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, context.imageEntity.url)) + val node = XpathUtils.getAsNode(document, IMG_XPATH) ?: throw HostException( String.format( "Xpath '%s' cannot be found in '%s'", IMG_XPATH, context.imageEntity.url ) ) - return try { - log.debug(String.format("Resolving name and image url for %s", context.imageEntity.url)) - val imgTitle = Optional.ofNullable(titleNode) - .map { node: Node -> node.textContent.trim { it <= ' ' } } - .orElseGet { getDefaultImageName(context.imageEntity.url) } - Pair(imgTitle, urlNode.attributes.getNamedItem("href").textContent.trim { it <= ' ' }) - } catch (e: Exception) { - throw HostException("Unexpected error occurred", e) - } + return Pair( + node.attributes.getNamedItem("alt").textContent.trim(), + node.attributes.getNamedItem("src").textContent.trim() + ) } companion object { - private const val TITLE_XPATH = "//span[contains(@class,'imagename')]" - private const val IMG_XPATH = "//a[@id='download']" + private const val IMG_XPATH = "//img[contains(@class,'img-fluid')]" } } \ No newline at end of file diff --git a/vripper-core/src/main/kotlin/me/vripper/services/DataAccessService.kt b/vripper-core/src/main/kotlin/me/vripper/services/DataAccessService.kt index 170e260..f8d0fb7 100644 --- a/vripper-core/src/main/kotlin/me/vripper/services/DataAccessService.kt +++ b/vripper-core/src/main/kotlin/me/vripper/services/DataAccessService.kt @@ -9,6 +9,7 @@ import me.vripper.data.repositories.ThreadRepository import me.vripper.entities.* import me.vripper.event.* import me.vripper.model.ErrorCount +import me.vripper.utilities.LoggerDelegate import me.vripper.utilities.PathUtils import me.vripper.utilities.PathUtils.sanitize import me.vripper.vgapi.PostItem @@ -26,6 +27,8 @@ internal class DataAccessService( private val eventBus: EventBus, ) { + private val log by LoggerDelegate() + private val postEntityIdCache: LoadingCache = Caffeine.newBuilder().expireAfterAccess(5, TimeUnit.MINUTES).build { id -> transaction { postRepository.findById(id) } @@ -40,9 +43,10 @@ internal class DataAccessService( val savedPost = postRepository.save(listOf(postEntity)).first() save(images.map { it.copy(postEntityId = savedPost.id) }) + // Publish event inside transaction for consistency + eventBus.publishEvent(PostCreateEvent(listOf(savedPost))) savedPost } - eventBus.publishEvent(PostCreateEvent(listOf(savedPost))) return savedPost } @@ -51,27 +55,34 @@ internal class DataAccessService( postEntities.forEach { postEntity -> postEntityIdCache.put(postEntity.id, postEntity) } + // Log and publish event after transaction commit but before returning + log.debug("[{}] Publishing event: PostUpdateEvent for {} posts", System.currentTimeMillis(), postEntities.size) eventBus.publishEvent(PostUpdateEvent(postEntities)) } fun updatePost(postEntity: PostEntity) { transaction { postRepository.update(postEntity) } postEntityIdCache.put(postEntity.id, postEntity) + // Log and publish event after transaction commit but before returning + log.debug("[{}] Publishing event: PostUpdateEvent for post {}", System.currentTimeMillis(), postEntity.id) eventBus.publishEvent(PostUpdateEvent(listOf(postEntity))) } fun save(threadEntity: ThreadEntity) { val savedThread = transaction { threadRepository.save(threadEntity) } + log.debug("[{}] Publishing event: ThreadCreateEvent for thread {}", System.currentTimeMillis(), savedThread.id) eventBus.publishEvent(ThreadCreateEvent(savedThread)) } fun update(threadEntity: ThreadEntity) { transaction { threadRepository.update(threadEntity) } + log.debug("[{}] Publishing event: ThreadUpdateEvent for thread {}", System.currentTimeMillis(), threadEntity.id) eventBus.publishEvent(ThreadUpdateEvent(threadEntity)) } fun updateImages(imageEntities: List) { transaction { imageRepository.update(imageEntities) } + log.debug("[{}] Publishing event: ImageEvent for {} images", System.currentTimeMillis(), imageEntities.size) eventBus.publishEvent(ImageEvent(imageEntities)) } @@ -81,6 +92,7 @@ internal class DataAccessService( imageRepository.update(imageEntity) } } + log.debug("[{}] Publishing event: ImageEvent for image {}", System.currentTimeMillis(), imageEntity.id) eventBus.publishEvent(ImageEvent(listOf(imageEntity))) } @@ -183,12 +195,15 @@ internal class DataAccessService( postEntityIdCache.get(postEntityId)?.let { postEntityIdCache.invalidate(it.id) } postEntityIdCache.invalidate(postEntityId) } + log.debug("[{}] Publishing event: PostDeleteEvent for {} posts", System.currentTimeMillis(), postEntityIds.size) eventBus.publishEvent(PostDeleteEvent(postEntityIds = postEntityIds)) + log.debug("[{}] Publishing event: ErrorCountEvent after post deletion", System.currentTimeMillis()) eventBus.publishEvent(ErrorCountEvent(ErrorCount(countImagesInError()))) } fun removeThread(threadId: Long) { transaction { threadRepository.deleteByThreadId(threadId) } + log.debug("[{}] Publishing event: ThreadDeleteEvent for thread {}", System.currentTimeMillis(), threadId) eventBus.publishEvent(ThreadDeleteEvent(threadId)) } @@ -216,11 +231,17 @@ internal class DataAccessService( fun saveMetadata(metadataEntity: MetadataEntity) { transaction { metadataRepository.save(metadataEntity) } + log.debug( + "[{}] Publishing event: MetadataUpdateEvent for post {}", + System.currentTimeMillis(), + metadataEntity.postIdRef + ) eventBus.publishEvent(MetadataUpdateEvent(metadataEntity)) } fun clearQueueLinks() { transaction { threadRepository.deleteAll() } + log.debug("[{}] Publishing event: ThreadClearEvent", System.currentTimeMillis()) eventBus.publishEvent(ThreadClearEvent()) } diff --git a/vripper-core/src/main/kotlin/me/vripper/services/DownloadSpeedService.kt b/vripper-core/src/main/kotlin/me/vripper/services/DownloadSpeedService.kt index c19a4ef..5c27605 100644 --- a/vripper-core/src/main/kotlin/me/vripper/services/DownloadSpeedService.kt +++ b/vripper-core/src/main/kotlin/me/vripper/services/DownloadSpeedService.kt @@ -6,6 +6,7 @@ import me.vripper.event.DownloadSpeedEvent import me.vripper.event.EventBus import me.vripper.event.QueueStateEvent import me.vripper.model.DownloadSpeed +import me.vripper.utilities.LoggerDelegate import java.util.concurrent.atomic.AtomicLong internal class DownloadSpeedService( @@ -16,6 +17,7 @@ internal class DownloadSpeedService( const val DOWNLOAD_POLL_RATE = 2500 } + private val log by LoggerDelegate() private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default) private val bytesCount = AtomicLong(0) private var job: Job? = null @@ -32,7 +34,13 @@ internal class DownloadSpeedService( while (isActive) { delay(DOWNLOAD_POLL_RATE.toLong()) val newValue = bytesCount.getAndSet(0) - eventBus.publishEvent(DownloadSpeedEvent(DownloadSpeed(((newValue * 1000) / DOWNLOAD_POLL_RATE)))) + val speed = DownloadSpeed(((newValue * 1000) / DOWNLOAD_POLL_RATE)) + log.debug( + "[{}] Publishing event: DownloadSpeedEvent({})", + System.currentTimeMillis(), + speed + ) + eventBus.publishEvent(DownloadSpeedEvent(speed)) } } } @@ -40,6 +48,7 @@ internal class DownloadSpeedService( job?.cancel() coroutineScope.launch { delay(DOWNLOAD_POLL_RATE + 500L) + log.debug("[{}] Publishing event: DownloadSpeedEvent(0)", System.currentTimeMillis()) eventBus.publishEvent(DownloadSpeedEvent(DownloadSpeed(0L))) } } diff --git a/vripper-core/src/main/kotlin/me/vripper/services/VGAuthService.kt b/vripper-core/src/main/kotlin/me/vripper/services/VGAuthService.kt index d9ea0b8..2e9fe4b 100644 --- a/vripper-core/src/main/kotlin/me/vripper/services/VGAuthService.kt +++ b/vripper-core/src/main/kotlin/me/vripper/services/VGAuthService.kt @@ -29,11 +29,14 @@ internal class VGAuthService( fun authenticate(settings: Settings) { if (!settings.viperSettings.login) { log.debug("Authentication option is disabled") - authenticated = false - loggedUser = "" + synchronized(this) { + authenticated = false + loggedUser = "" + } synchronized(vgCookies) { vgCookies.clear() } + log.debug("[{}] Publishing event: VGUserLoginEvent", System.currentTimeMillis()) eventBus.publishEvent(VGUserLoginEvent(loggedUser)) return } @@ -41,11 +44,14 @@ internal class VGAuthService( val password = settings.viperSettings.password if (username.isEmpty() || password.isEmpty()) { log.error("Cannot authenticate with ViperGirls credentials, username or password is empty") - authenticated = false - loggedUser = "" + synchronized(this) { + authenticated = false + loggedUser = "" + } synchronized(vgCookies) { vgCookies.clear() } + log.debug("[{}] Publishing event: VGUserLoginEvent (empty credentials)", System.currentTimeMillis()) eventBus.publishEvent(VGUserLoginEvent(loggedUser)) return } @@ -82,6 +88,7 @@ internal class VGAuthService( "Failed to authenticate user with {}, missing vg_userid/vg_password cookie", settings.viperSettings.host ) + log.debug("[${System.currentTimeMillis()}] Publishing event: VGUserLoginEvent (missing cookies)") eventBus.publishEvent(VGUserLoginEvent(loggedUser)) return } @@ -98,15 +105,21 @@ internal class VGAuthService( log.error( "Failed to authenticate user with " + settings.viperSettings.host, e ) - authenticated = false - loggedUser = "" + synchronized(this) { + authenticated = false + loggedUser = "" + } + log.debug("[{}] Publishing event: VGUserLoginEvent (exception)", System.currentTimeMillis()) eventBus.publishEvent(VGUserLoginEvent(loggedUser)) return } - authenticated = true - loggedUser = username + synchronized(this) { + authenticated = true + loggedUser = username + } + log.debug("[{}] Publishing event: VGUserLoginEvent for user {}", System.currentTimeMillis(), username) eventBus.publishEvent(VGUserLoginEvent(loggedUser)) - log.info("Successfully logged in as: $loggedUser") + log.info("Successfully logged in as: {}", loggedUser) } fun leaveThanks(postEntity: PostEntity) { diff --git a/vripper-core/src/main/kotlin/me/vripper/services/download/DownloadService.kt b/vripper-core/src/main/kotlin/me/vripper/services/download/DownloadService.kt index 835dd01..d56aa25 100644 --- a/vripper-core/src/main/kotlin/me/vripper/services/download/DownloadService.kt +++ b/vripper-core/src/main/kotlin/me/vripper/services/download/DownloadService.kt @@ -54,12 +54,20 @@ internal class DownloadService( } fun stop(postEntityIds: List = emptyList()) { - if (postEntityIds.isNotEmpty()) { - stopInternal(postEntityIds) - eventBus.publishEvent(StoppedEvent(postEntityIds)) - } else { - stopAll() - eventBus.publishEvent(StoppedEvent(listOf(-1))) + downloadManagerLock.withLock { + if (postEntityIds.isNotEmpty()) { + stopInternal(postEntityIds) + log.debug( + "[{}] Publishing event: StoppedEvent for {} posts", + System.currentTimeMillis(), + postEntityIds.size + ) + eventBus.publishEvent(StoppedEvent(postEntityIds)) + } else { + stopAll() + log.debug("[{}] Publishing event: StoppedEvent for all posts", System.currentTimeMillis()) + eventBus.publishEvent(StoppedEvent(listOf(-1))) + } } } @@ -125,24 +133,20 @@ internal class DownloadService( } private fun stopAll() { - downloadManagerLock.withLock { - queueManager.clearPending() - queueManager.clearRunning() - dataAccessService.findAllNonCompletedPostEntityIds().forEach { - dataAccessService.stopImagesByPostEntityIdAndIsNotCompleted(it) - dataAccessService.finishPost(it) - } + queueManager.clearPending() + queueManager.clearRunning() + dataAccessService.findAllNonCompletedPostEntityIds().forEach { + dataAccessService.stopImagesByPostEntityIdAndIsNotCompleted(it) + dataAccessService.finishPost(it) } } private fun stopInternal(postEntityIds: List) { - downloadManagerLock.withLock { - postEntityIds.forEach { - queueManager.clearPending(it) - queueManager.clearRunning(it) - dataAccessService.stopImagesByPostEntityIdAndIsNotCompleted(it) - dataAccessService.finishPost(it) - } + postEntityIds.forEach { + queueManager.clearPending(it) + queueManager.clearRunning(it) + dataAccessService.stopImagesByPostEntityIdAndIsNotCompleted(it) + dataAccessService.finishPost(it) } } diff --git a/vripper-core/src/main/kotlin/me/vripper/services/download/QueueManager.kt b/vripper-core/src/main/kotlin/me/vripper/services/download/QueueManager.kt index 868f7a1..cc953ee 100644 --- a/vripper-core/src/main/kotlin/me/vripper/services/download/QueueManager.kt +++ b/vripper-core/src/main/kotlin/me/vripper/services/download/QueueManager.kt @@ -66,19 +66,21 @@ internal class QueueManager( } fun accept(accepted: List) { - pending.forEach { - it.removeAll(accepted) + downloadManagerLock.withLock { + pending.forEach { + it.removeAll(accepted) + } + pending.removeIf { it.isEmpty() } + accepted.map { + ImageDownloadRunnable( + dataAccessService.findImageById(it.imageEntityId).orElseThrow(), settingsService.settings.copy() + ) + }.forEach { + launch(it) + running.add(it) + } + reportQueueState() } - pending.removeIf { it.isEmpty() } - accepted.map { - ImageDownloadRunnable( - dataAccessService.findImageById(it.imageEntityId).orElseThrow(), settingsService.settings.copy() - ) - }.forEach { - launch(it) - running.add(it) - } - reportQueueState() } fun pending(): List { @@ -121,8 +123,8 @@ internal class QueueManager( } } } + reportQueueState() } - reportQueueState() } fun getQueueState(): QueueState { @@ -146,8 +148,6 @@ internal class QueueManager( dataAccessService.updateImage(image) }.onComplete { afterJobFinish(runnable) - reportQueueState() - eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataAccessService.countImagesInError()))) log.debug( "Finished downloading ${runnable.context.imageEntity.url}" ) @@ -161,12 +161,21 @@ internal class QueueManager( if (!isPending(image.postEntityId) && !isRunning(image.postEntityId) && !imageDownloadRunnable.stopped) { dataAccessService.finishPost(image.postEntityId, true) } + reportQueueState() + log.debug("[{}] Event published: ErrorCountEvent after job finish", System.currentTimeMillis()) + eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataAccessService.countImagesInError()))) downloadManagerCondition.signal() } } private fun reportQueueState() { val queueState = getQueueState() + log.debug( + "[{}] Publishing event: QueueStateEvent(running={}, remaining={})", + System.currentTimeMillis(), + queueState.running, + queueState.remaining + ) eventBus.publishEvent(QueueStateEvent(queueState)) } } \ No newline at end of file diff --git a/vripper-gui/src/main/kotlin/me/vripper/gui/components/views/PostInfoView.kt b/vripper-gui/src/main/kotlin/me/vripper/gui/components/views/PostInfoView.kt index c667aec..5ada087 100644 --- a/vripper-gui/src/main/kotlin/me/vripper/gui/components/views/PostInfoView.kt +++ b/vripper-gui/src/main/kotlin/me/vripper/gui/components/views/PostInfoView.kt @@ -18,7 +18,7 @@ class PostInfoView : View() { private val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO) private val imagesTableView: ImagesTableView by inject() private val postModel: PostModel = PostModel( - -1, -1, "", 0.0, "", "", 0, 0, "", "", "*", "", "", "", emptyList(), emptyList(), "", 0 + -1, -1, "", 0.0, "", "", 0, 0, "", "", -1, "", "", "", emptyList(), emptyList(), "", 0 ) override val root = tabpane() @@ -89,7 +89,7 @@ class PostInfoView : View() { this.total = 0 this.hosts = "" this.addedOn = "" - this.order = "*" + this.order = -1 this.path = "" this.folderName = "" this.progressCount = "" diff --git a/vripper-gui/src/main/kotlin/me/vripper/gui/components/views/PostsTableView.kt b/vripper-gui/src/main/kotlin/me/vripper/gui/components/views/PostsTableView.kt index b2b7ee1..49ad7f8 100644 --- a/vripper-gui/src/main/kotlin/me/vripper/gui/components/views/PostsTableView.kt +++ b/vripper-gui/src/main/kotlin/me/vripper/gui/components/views/PostsTableView.kt @@ -203,7 +203,38 @@ class PostsTableView : View() { } } cellFactory = Callback { - TextFieldTableCell().apply { alignment = Pos.CENTER_LEFT } + object : TableCell() { + override fun updateItem(item: Number?, empty: Boolean) { + super.updateItem(item, empty) + text = when { + empty -> null + item == null -> null + item.toLong() == -1L -> "*" // Display "*" for -1 + else -> item.toString() // Display the actual value + } + alignment = Pos.CENTER_LEFT + } + } + } + comparator = Comparator { a, b -> + when { + a == null && b == null -> 0 + a == null -> 1 // Nulls to bottom + b == null -> -1 + a.toLong() == -1L && b.toLong() == -1L -> 0 // Both -1, equal + a.toLong() == -1L -> { + // -1 always goes to bottom + // In ascending: return 1 (a is greater, goes last) + // In descending: return -1 (a is smaller, still goes last in reverse) + if (sortType == TableColumn.SortType.ASCENDING) 1 else -1 + } + + b.toLong() == -1L -> { + if (sortType == TableColumn.SortType.ASCENDING) -1 else 1 + } + + else -> a.toLong().compareTo(b.toLong()) // Normal numeric comparison + } } } column("Preview", PostModel::previewListProperty) { @@ -496,11 +527,11 @@ class PostsTableView : View() { val rank = queueState.rank.find { it.postEntityId == post.id } if (rank != null) { runLater { - post.order = rank.rank.toString() + post.order = rank.rank } - } else if (post.order != "*") { + } else if (post.order != -1L) { runLater { - post.order = "*" + post.order = -1L } } } diff --git a/vripper-gui/src/main/kotlin/me/vripper/gui/controller/PostController.kt b/vripper-gui/src/main/kotlin/me/vripper/gui/controller/PostController.kt index d630a2f..1d818ad 100644 --- a/vripper-gui/src/main/kotlin/me/vripper/gui/controller/PostController.kt +++ b/vripper-gui/src/main/kotlin/me/vripper/gui/controller/PostController.kt @@ -115,7 +115,7 @@ class PostController : Controller() { post.total, post.hosts.joinToString(separator = ", "), post.addedOn.format(dateTimeFormatter), - "*", + -1, post.getDownloadFolder(), post.folderName, progressCount(post.total, post.done, post.downloaded), diff --git a/vripper-gui/src/main/kotlin/me/vripper/gui/model/PostModel.kt b/vripper-gui/src/main/kotlin/me/vripper/gui/model/PostModel.kt index 594c262..5876845 100644 --- a/vripper-gui/src/main/kotlin/me/vripper/gui/model/PostModel.kt +++ b/vripper-gui/src/main/kotlin/me/vripper/gui/model/PostModel.kt @@ -1,9 +1,6 @@ package me.vripper.gui.model -import javafx.beans.property.SimpleDoubleProperty -import javafx.beans.property.SimpleIntegerProperty -import javafx.beans.property.SimpleListProperty -import javafx.beans.property.SimpleStringProperty +import javafx.beans.property.* import javafx.collections.FXCollections import javafx.collections.ObservableList import tornadofx.getValue @@ -20,7 +17,7 @@ class PostModel( total: Int, hosts: String, addedOn: String, - order: String, + order: Long, path: String, folderName: String, progressCount: String, @@ -54,8 +51,8 @@ class PostModel( val addedOnProperty = SimpleStringProperty(addedOn) var addedOn: String by addedOnProperty - val orderProperty = SimpleStringProperty(order) - var order: String by orderProperty + val orderProperty = SimpleLongProperty(order) + var order: Long by orderProperty val pathProperty = SimpleStringProperty(path) var path: String by pathProperty