Compare commits

...
Author SHA1 Message Date
dev-claw 84f90fd40c fixes #223 2025-03-21 16:34:25 +01:00
40 changed files with 409 additions and 340 deletions
+6 -6
View File
@@ -31,13 +31,13 @@
<kotlinx-serialization-json.version>1.7.3</kotlinx-serialization-json.version>
<httpclient5.version>5.4</httpclient5.version>
<httpcore5.version>5.3</httpcore5.version>
<commons-codec.version>1.17.1</commons-codec.version>
<liquibase-core.version>4.29.2</liquibase-core.version>
<liquibase-slf4j.version>5.0.0</liquibase-slf4j.version>
<grpc.kotlin.version>1.4.1</grpc.kotlin.version>
<java.grpc.version>1.68.1</java.grpc.version>
<protobuf.version>3.25.5</protobuf.version>
<sqlite-jdbc.version>3.47.0.0</sqlite-jdbc.version>
<bucket4j_jdk17-core.version>8.14.0</bucket4j_jdk17-core.version>
</properties>
<build>
<plugins>
@@ -186,11 +186,6 @@
<groupId>org.apache.httpcomponents.core5</groupId>
<version>${httpcore5.version}</version>
</dependency>
<dependency>
<artifactId>commons-codec</artifactId>
<groupId>commons-codec</groupId>
<version>${commons-codec.version}</version>
</dependency>
<dependency>
<artifactId>liquibase-core</artifactId>
<groupId>org.liquibase</groupId>
@@ -221,6 +216,11 @@
<artifactId>protobuf-kotlin</artifactId>
<version>${protobuf.version}</version>
</dependency>
<dependency>
<groupId>com.bucket4j</groupId>
<artifactId>bucket4j_jdk17-core</artifactId>
<version>${bucket4j_jdk17-core.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
</project>
-5
View File
@@ -75,10 +75,6 @@
<artifactId>caffeine</artifactId>
<groupId>com.github.ben-manes.caffeine</groupId>
</dependency>
<dependency>
<artifactId>commons-codec</artifactId>
<groupId>commons-codec</groupId>
</dependency>
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna-platform</artifactId>
@@ -111,7 +107,6 @@
<dependency>
<groupId>com.bucket4j</groupId>
<artifactId>bucket4j_jdk17-core</artifactId>
<version>8.14.0</version>
</dependency>
</dependencies>
@@ -8,7 +8,6 @@ import me.vripper.data.repositories.impl.ImageRepositoryImpl
import me.vripper.data.repositories.impl.MetadataRepositoryImpl
import me.vripper.data.repositories.impl.PostRepositoryImpl
import me.vripper.data.repositories.impl.ThreadRepositoryImpl
import me.vripper.download.DownloadService
import me.vripper.event.EventBus
import me.vripper.host.*
import me.vripper.services.*
@@ -45,16 +44,16 @@ val coreModule = module {
RetryPolicyService(get(), get())
}
single<HTTPService> {
HTTPService(get(), get())
HTTPService(get())
}
single<VGAuthService> {
VGAuthService(get(), get(), get())
VGAuthService(get(), get())
}
single<ThreadCacheService> {
ThreadCacheService(get())
ThreadCacheService(get(), get())
}
single<DownloadService> {
DownloadService(get(), get(), get(), get(), get())
DownloadService(get(), get(), get(), get())
}
single<DownloadSpeedService> {
DownloadSpeedService(get())
@@ -1,29 +0,0 @@
package me.vripper.download
import kotlinx.coroutines.*
import me.vripper.entities.ImageEntity
import me.vripper.model.Settings
import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase
import org.apache.hc.client5.http.cookie.BasicCookieStore
import org.apache.hc.client5.http.protocol.HttpClientContext
import org.koin.core.component.KoinComponent
internal class ImageDownloadContext(val imageEntity: ImageEntity, val settings: Settings) : KoinComponent {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val jobs = mutableListOf<Job>()
val httpContext: HttpClientContext =
HttpClientContext.create().apply { cookieStore = BasicCookieStore() }
val requests = mutableListOf<HttpUriRequestBase>()
val postId = imageEntity.postIdRef
fun cancelCoroutines() {
runBlocking {
coroutineScope.cancel()
jobs.forEach { job -> job.cancelAndJoin() }
}
}
fun launchCoroutine(block: suspend CoroutineScope.() -> Unit): Job {
return coroutineScope.launch(block = block).also { job -> jobs.add(job) }
}
}
@@ -1,154 +0,0 @@
package me.vripper.download
import dev.failsafe.function.CheckedRunnable
import me.vripper.entities.ImageEntity
import me.vripper.entities.Status
import me.vripper.exception.DownloadException
import me.vripper.exception.HostException
import me.vripper.host.DownloadedImage
import me.vripper.host.Host
import me.vripper.host.ImageMimeType
import me.vripper.model.Settings
import me.vripper.services.DataTransaction
import me.vripper.services.VGAuthService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.PathUtils.getExtension
import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
import me.vripper.utilities.PathUtils.sanitize
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.io.IOException
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardCopyOption
import java.util.*
import kotlin.io.path.Path
import kotlin.io.path.pathString
internal class ImageDownloadRunnable(
val imageEntity: ImageEntity, val postRank: Int, private val settings: Settings
) : KoinComponent, CheckedRunnable {
private val log by LoggerDelegate()
private val dataTransaction: DataTransaction by inject()
private val vgauthService: VGAuthService by inject()
private val hosts: List<Host> = getKoin().getAll()
var completed = false
var stopped = false
private lateinit var context: ImageDownloadContext
fun download() {
try {
imageEntity.status = Status.DOWNLOADING
imageEntity.downloaded = 0
dataTransaction.updateImage(imageEntity)
synchronized(imageEntity.postId.toString().intern()) {
val post = dataTransaction.findPostById(context.postId)
if (post.status != Status.DOWNLOADING) {
post.status = Status.DOWNLOADING
dataTransaction.updatePost(post)
vgauthService.leaveThanks(post)
}
}
log.debug("Getting image url and name from ${imageEntity.url} using ${imageEntity.host}")
val host = hosts.first { it.isSupported(imageEntity.url) }
val downloadedImage = host.downloadInternal(imageEntity.url, context)
log.debug("Resolved name for ${imageEntity.url}: ${downloadedImage.name}")
log.debug("Downloaded image {} to {}", imageEntity.url, downloadedImage.path)
synchronized(imageEntity.postId.toString().intern()) {
val post = dataTransaction.findPostById(context.postId)
val downloadDirectory = Path(post.downloadDirectory, post.folderName).pathString
checkImageTypeAndRename(
downloadDirectory, downloadedImage, imageEntity.index
)
if (imageEntity.downloaded == imageEntity.size && imageEntity.size > 0) {
imageEntity.status = Status.FINISHED
post.done += 1
post.downloaded += imageEntity.size
dataTransaction.updatePost(post)
} else {
imageEntity.status = Status.ERROR
}
dataTransaction.updateImage(imageEntity)
}
} catch (e: Exception) {
if (stopped) {
return
}
imageEntity.status = Status.ERROR
dataTransaction.updateImage(imageEntity)
throw DownloadException(e)
}
}
@Throws(HostException::class)
private fun checkImageTypeAndRename(
downloadDirectory: String, downloadedImage: DownloadedImage, index: Int
) {
val existingExtension = getExtension(downloadedImage.name).lowercase()
val fileNameWithoutExtension = getFileNameWithoutExtension(downloadedImage.name)
val extension = when (downloadedImage.type) {
ImageMimeType.IMAGE_BMP -> "BMP"
ImageMimeType.IMAGE_GIF -> "GIF"
ImageMimeType.IMAGE_JPEG -> "JPG"
ImageMimeType.IMAGE_PNG -> "PNG"
ImageMimeType.IMAGE_WEBP -> "WEBP"
}
val filename =
if (existingExtension.isBlank()) "${sanitize(downloadedImage.name)}.$extension" else "${
sanitize(
fileNameWithoutExtension
)
}.$extension"
try {
val downloadDestinationFolder = Path.of(downloadDirectory)
Files.createDirectories(downloadDestinationFolder)
val finalFilename = "${
if (settings.downloadSettings.forceOrder) String.format(
"%03d_", index + 1
) else ""
}$filename"
imageEntity.filename = finalFilename
val imageDownloadPath = downloadDestinationFolder.resolve(finalFilename)
Files.copy(downloadedImage.path, imageDownloadPath, StandardCopyOption.REPLACE_EXISTING)
} catch (e: Exception) {
throw HostException("Failed to rename the image", e)
} finally {
try {
Files.delete(downloadedImage.path)
} catch (_: IOException) {
}
}
}
override fun run() {
context = ImageDownloadContext(imageEntity, settings)
try {
if (stopped) {
return
}
download()
} finally {
completed = true
context.cancelCoroutines()
}
}
fun stop() {
stopped = true
context.requests.forEach { it.abort() }
context.cancelCoroutines()
dataTransaction.updateImage(context.imageEntity)
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other == null || javaClass != other.javaClass) return false
val that = other as ImageDownloadRunnable
return imageEntity.id == that.imageEntity.id
}
override fun hashCode(): Int {
return Objects.hash(imageEntity.id)
}
}
@@ -1,9 +1,9 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.utilities.HtmlUtils
@@ -1,9 +1,11 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
@@ -4,10 +4,10 @@ import kotlinx.coroutines.cancelAndJoin
import kotlinx.coroutines.delay
import kotlinx.coroutines.isActive
import kotlinx.coroutines.runBlocking
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.DownloadException
import me.vripper.exception.HostException
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.utilities.HtmlUtils
@@ -22,7 +22,6 @@ import org.w3c.dom.Document
import java.io.BufferedOutputStream
import java.nio.file.Files
import java.nio.file.Path
import kotlin.Throws
internal abstract class Host(
val hostName: String,
@@ -1,9 +1,11 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
@@ -1,9 +1,11 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
@@ -1,9 +1,11 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
@@ -1,9 +1,11 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
@@ -1,9 +1,11 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
@@ -1,9 +1,11 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
@@ -1,10 +1,12 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.HtmlProcessorException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
@@ -1,9 +1,11 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
@@ -1,9 +1,11 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
@@ -1,9 +1,11 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
@@ -1,9 +1,11 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
@@ -1,9 +1,11 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
@@ -1,9 +1,11 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
@@ -1,9 +1,11 @@
package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadService.ImageDownloadContext
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
import org.w3c.dom.Document
@@ -1,17 +1,30 @@
package me.vripper.listeners
import me.vripper.services.DataTransaction
import me.vripper.services.MetadataService
import me.vripper.services.*
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
open class OnStartupListener : KoinComponent {
private val dataTransaction: DataTransaction by inject()
private val metadataService: MetadataService by inject()
private val settingsService: SettingsService by inject()
private val vgAuthService: VGAuthService by inject()
private val downloadSpeedService: DownloadSpeedService by inject()
private val httpService: HTTPService by inject()
private val retryPolicyService: RetryPolicyService by inject()
private val threadCacheService: ThreadCacheService by inject()
private val downloadService: DownloadService by inject()
open fun run() {
dataTransaction.setDownloadingToStopped()
dataTransaction.stopImagesByPostIdAndIsNotCompleted()
vgAuthService.init()
httpService.init()
retryPolicyService.init()
threadCacheService.init()
settingsService.init()
metadataService.init()
downloadSpeedService.init()
downloadService.init()
}
}
@@ -3,7 +3,6 @@ package me.vripper.services
import kotlinx.coroutines.flow.*
import kotlinx.coroutines.time.sample
import kotlinx.serialization.json.Json
import me.vripper.download.DownloadService
import me.vripper.entities.*
import me.vripper.event.*
import me.vripper.exception.PostParseException
@@ -1,8 +1,9 @@
package me.vripper.download
package me.vripper.services
import dev.failsafe.Failsafe
import dev.failsafe.RetryPolicy
import kotlinx.coroutines.Runnable
import dev.failsafe.function.CheckedRunnable
import kotlinx.coroutines.*
import me.vripper.entities.ImageEntity
import me.vripper.entities.PostEntity
import me.vripper.entities.Status
@@ -10,24 +11,39 @@ import me.vripper.event.ErrorCountEvent
import me.vripper.event.EventBus
import me.vripper.event.QueueStateEvent
import me.vripper.event.StoppedEvent
import me.vripper.exception.DownloadException
import me.vripper.exception.HostException
import me.vripper.host.DownloadedImage
import me.vripper.host.Host
import me.vripper.host.ImageMimeType
import me.vripper.model.ErrorCount
import me.vripper.model.QueueState
import me.vripper.services.DataTransaction
import me.vripper.services.RetryPolicyService
import me.vripper.services.SettingsService
import me.vripper.services.VGAuthService
import me.vripper.model.Settings
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.PathUtils.getExtension
import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
import me.vripper.utilities.PathUtils.sanitize
import me.vripper.utilities.downloadRunner
import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase
import org.apache.hc.client5.http.cookie.BasicCookieStore
import org.apache.hc.client5.http.protocol.HttpClientContext
import org.jetbrains.exposed.sql.transactions.transaction
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.io.IOException
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardCopyOption
import java.util.*
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
import kotlin.io.path.Path
import kotlin.io.path.pathString
internal class DownloadService(
private val settingsService: SettingsService,
private val dataTransaction: DataTransaction,
private val retryPolicyService: RetryPolicyService,
private val vgAuthService: VGAuthService,
private val eventBus: EventBus
) {
private val maxPoolSize: Int = 24
@@ -37,7 +53,155 @@ internal class DownloadService(
private val lock = ReentrantLock()
private val condition = lock.newCondition()
init {
internal class ImageDownloadContext(val imageEntity: ImageEntity, val settings: Settings) : KoinComponent {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val jobs = mutableListOf<Job>()
val httpContext: HttpClientContext =
HttpClientContext.create().apply { cookieStore = BasicCookieStore() }
val requests = mutableListOf<HttpUriRequestBase>()
val postId = imageEntity.postIdRef
fun cancelCoroutines() {
runBlocking {
coroutineScope.cancel()
jobs.forEach { job -> job.cancelAndJoin() }
}
}
fun launchCoroutine(block: suspend CoroutineScope.() -> Unit): Job {
return coroutineScope.launch(block = block).also { job -> jobs.add(job) }
}
}
internal class ImageDownloadRunnable(
val imageEntity: ImageEntity, val postRank: Int, private val settings: Settings
) : KoinComponent, CheckedRunnable {
private val log by LoggerDelegate()
private val dataTransaction: DataTransaction by inject()
private val vgauthService: VGAuthService by inject()
private val hosts: List<Host> = getKoin().getAll()
var completed = false
var stopped = false
private lateinit var context: ImageDownloadContext
fun download() {
try {
imageEntity.status = Status.DOWNLOADING
imageEntity.downloaded = 0
dataTransaction.updateImage(imageEntity)
synchronized(imageEntity.postId.toString().intern()) {
val post = dataTransaction.findPostById(context.postId)
if (post.status != Status.DOWNLOADING) {
post.status = Status.DOWNLOADING
dataTransaction.updatePost(post)
vgauthService.leaveThanks(post)
}
}
log.debug("Getting image url and name from ${imageEntity.url} using ${imageEntity.host}")
val host = hosts.first { it.isSupported(imageEntity.url) }
val downloadedImage = host.downloadInternal(imageEntity.url, context)
log.debug("Resolved name for ${imageEntity.url}: ${downloadedImage.name}")
log.debug("Downloaded image {} to {}", imageEntity.url, downloadedImage.path)
synchronized(imageEntity.postId.toString().intern()) {
val post = dataTransaction.findPostById(context.postId)
val downloadDirectory = Path(post.downloadDirectory, post.folderName).pathString
checkImageTypeAndRename(
downloadDirectory, downloadedImage, imageEntity.index
)
if (imageEntity.downloaded == imageEntity.size && imageEntity.size > 0) {
imageEntity.status = Status.FINISHED
post.done += 1
post.downloaded += imageEntity.size
dataTransaction.updatePost(post)
} else {
imageEntity.status = Status.ERROR
}
dataTransaction.updateImage(imageEntity)
}
} catch (e: Exception) {
if (stopped) {
return
}
imageEntity.status = Status.ERROR
dataTransaction.updateImage(imageEntity)
throw DownloadException(e)
}
}
@Throws(HostException::class)
private fun checkImageTypeAndRename(
downloadDirectory: String, downloadedImage: DownloadedImage, index: Int
) {
val existingExtension = getExtension(downloadedImage.name).lowercase()
val fileNameWithoutExtension = getFileNameWithoutExtension(downloadedImage.name)
val extension = when (downloadedImage.type) {
ImageMimeType.IMAGE_BMP -> "BMP"
ImageMimeType.IMAGE_GIF -> "GIF"
ImageMimeType.IMAGE_JPEG -> "JPG"
ImageMimeType.IMAGE_PNG -> "PNG"
ImageMimeType.IMAGE_WEBP -> "WEBP"
}
val filename =
if (existingExtension.isBlank()) "${sanitize(downloadedImage.name)}.$extension" else "${
sanitize(
fileNameWithoutExtension
)
}.$extension"
try {
val downloadDestinationFolder = Path.of(downloadDirectory)
Files.createDirectories(downloadDestinationFolder)
val finalFilename = "${
if (settings.downloadSettings.forceOrder) String.format(
"%03d_", index + 1
) else ""
}$filename"
imageEntity.filename = finalFilename
val imageDownloadPath = downloadDestinationFolder.resolve(finalFilename)
Files.copy(downloadedImage.path, imageDownloadPath, StandardCopyOption.REPLACE_EXISTING)
} catch (e: Exception) {
throw HostException("Failed to rename the image", e)
} finally {
try {
Files.delete(downloadedImage.path)
} catch (_: IOException) {
}
}
}
override fun run() {
context = ImageDownloadContext(imageEntity, settings)
try {
if (stopped) {
return
}
download()
} finally {
completed = true
context.cancelCoroutines()
}
}
fun stop() {
stopped = true
context.requests.forEach { it.abort() }
context.cancelCoroutines()
dataTransaction.updateImage(context.imageEntity)
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (other == null || javaClass != other.javaClass) return false
val that = other as ImageDownloadRunnable
return imageEntity.id == that.imageEntity.id
}
override fun hashCode(): Int {
return Objects.hash(imageEntity.id)
}
}
fun init() {
Thread.ofVirtual().name("Download Loop").unstarted(Runnable {
val accepted: MutableList<ImageDownloadRunnable> = mutableListOf()
val candidates: MutableList<ImageDownloadRunnable> = mutableListOf()
@@ -59,7 +223,7 @@ internal class DownloadService(
candidates.clear()
try {
condition.await()
} catch (e: InterruptedException) {
} catch (_: InterruptedException) {
Thread.currentThread().interrupt()
}
}
@@ -189,7 +353,7 @@ internal class DownloadService(
private fun candidateCount(): Map<Byte, Int> {
val map: MutableMap<Byte, Int> = mutableMapOf()
Host.getHosts().values.forEach { host: Byte ->
Host.Companion.getHosts().values.forEach { host: Byte ->
val imageDownloadRunnableList: List<ImageDownloadRunnable> = running.computeIfAbsent(
host
) { mutableListOf() }
@@ -20,7 +20,7 @@ internal class DownloadSpeedService(
private val bytesCount = AtomicLong(0)
private var job: Job? = null
init {
fun init() {
coroutineScope.launch {
eventBus.events.filterIsInstance(QueueStateEvent::class).collect {
if (it.queueState.running + it.queueState.remaining > 0) {
@@ -10,15 +10,15 @@ import org.apache.hc.client5.http.cookie.StandardCookieSpec
import org.apache.hc.client5.http.impl.DefaultRedirectStrategy
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient
import org.apache.hc.client5.http.impl.classic.HttpClients
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager
import org.apache.hc.client5.http.impl.io.BasicHttpClientConnectionManager
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder
import org.apache.hc.client5.http.io.HttpClientConnectionManager
import org.apache.hc.core5.pool.PoolConcurrencyPolicy
import org.apache.hc.core5.pool.PoolReusePolicy
import org.apache.hc.core5.util.Timeout
internal class HTTPService(
private val eventBus: EventBus,
settingsService: SettingsService
private val eventBus: EventBus
) {
companion object {
@@ -28,23 +28,14 @@ internal class HTTPService(
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private lateinit var pcm: PoolingHttpClientConnectionManager
private lateinit var rc: RequestConfig
private lateinit var cc: ConnectionConfig
lateinit var client: CloseableHttpClient
private var connectionTimeout = settingsService.settings.connectionSettings.timeout
private var pcm: HttpClientConnectionManager = BasicHttpClientConnectionManager()
private var rc: RequestConfig = RequestConfig.DEFAULT
private var cc: ConnectionConfig = ConnectionConfig.DEFAULT
private var connectionTimeout = 30
private var connectionExpiryJob: Job? = null
var client: CloseableHttpClient = HttpClients.createDefault()
init {
buildRequestConfig()
buildConnectionConfig()
buildConnectionPool()
buildClientBuilder()
coroutineScope.launch {
while (isActive) {
pcm.closeExpired()
delay(60_000)
}
}
fun init() {
coroutineScope.launch {
eventBus
.events
@@ -64,13 +55,21 @@ internal class HTTPService(
}
private fun buildConnectionPool() {
connectionExpiryJob?.cancel()
pcm = PoolingHttpClientConnectionManagerBuilder.create()
.setPoolConcurrencyPolicy(PoolConcurrencyPolicy.LAX)
.setConnPoolPolicy(PoolReusePolicy.FIFO)
.setDefaultConnectionConfig(cc)
.setMaxConnTotal(Int.MAX_VALUE)
.setMaxConnPerRoute(Int.MAX_VALUE)
.build()
.build().also {
connectionExpiryJob = coroutineScope.launch {
while (isActive) {
it.closeExpired()
delay(60_000)
}
}
}
}
private fun buildRequestConfig() {
@@ -95,6 +94,5 @@ internal class HTTPService(
.disableAutomaticRetries()
.setDefaultRequestConfig(rc)
.build()
}
}
@@ -18,7 +18,7 @@ internal class RetryPolicyService(
private var maxAttempts: Int = settingsService.settings.connectionSettings.maxAttempts
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
init {
fun init() {
coroutineScope.launch {
eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collect {
if (maxAttempts != it.settings.connectionSettings.maxAttempts) {
@@ -1,5 +1,6 @@
package me.vripper.services
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.decodeFromStream
@@ -9,7 +10,7 @@ import me.vripper.exception.ValidationException
import me.vripper.model.Settings
import me.vripper.utilities.ApplicationProperties.VRIPPER_DIR
import me.vripper.utilities.LoggerDelegate
import org.apache.commons.codec.digest.DigestUtils
import me.vripper.utilities.md5Hex
import java.io.FileWriter
import java.nio.file.*
import kotlin.io.path.readText
@@ -28,16 +29,13 @@ class SettingsService(private val eventBus: EventBus) {
}
var settings = Settings()
init {
init()
}
private fun init() {
fun init() {
loadViperProxies()
restore()
eventBus.publishEvent(SettingsUpdateEvent(settings))
}
@OptIn(ExperimentalSerializationApi::class)
private fun loadViperProxies() {
try {
SettingsService::class.java.getResourceAsStream("/proxies.json")?.use {
@@ -84,7 +82,7 @@ class SettingsService(private val eventBus: EventBus) {
check(settings)
val viperSettings = if (settings.viperSettings.login) {
if (this.settings.viperSettings.password != settings.viperSettings.password) {
settings.viperSettings.copy(password = DigestUtils.md5Hex(settings.viperSettings.password))
settings.viperSettings.copy(password = md5Hex(settings.viperSettings.password))
} else {
settings.viperSettings
}
@@ -137,7 +135,7 @@ class SettingsService(private val eventBus: EventBus) {
fun check(settings: Settings) {
val path: Path = try {
Paths.get(settings.downloadSettings.downloadPath)
} catch (e: InvalidPathException) {
} catch (_: InvalidPathException) {
throw ValidationException(
String.format(
"%s is invalid", settings.downloadSettings.downloadPath
@@ -14,7 +14,7 @@ import me.vripper.vgapi.ThreadLookupAPIParser
import java.util.concurrent.ExecutionException
import java.util.concurrent.TimeUnit
internal class ThreadCacheService(val eventBus: EventBus) {
internal class ThreadCacheService(val eventBus: EventBus, val dataTransaction: DataTransaction) {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
@@ -28,7 +28,13 @@ internal class ThreadCacheService(val eventBus: EventBus) {
private val cache: LoadingCache<Long, ThreadItem> =
Caffeine.newBuilder().expireAfterWrite(20, TimeUnit.MINUTES).build { threadId ->
ThreadLookupAPIParser(threadId).parse()
val threadItem = ThreadLookupAPIParser(threadId).parse()
dataTransaction.findThreadByThreadId(threadItem.threadId).ifPresent {
if (threadItem.postItemList.isNotEmpty()) {
dataTransaction.update(it.copy(total = threadItem.postItemList.size))
}
}
threadItem
}
@Throws(ExecutionException::class)
@@ -25,49 +25,30 @@ import org.apache.hc.core5.http.message.BasicNameValuePair
internal class VGAuthService(
private val cm: HTTPService,
private val settingsService: SettingsService,
private val eventBus: EventBus
private val eventBus: EventBus,
) {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val log by LoggerDelegate()
val context: HttpClientContext = HttpClientContext.create()
private val vgCookies: MutableList<Cookie> = mutableListOf()
var loggedUser = ""
var authenticated = false
val clickCookies: List<Cookie>
get() {
return if (authenticated) {
val useridCookie = context.cookieStore.cookies.first { it.name.equals("vg_userid") }.let { cookie ->
BasicClientCookie(cookie.name, cookie.value).apply {
domain = "viper.click"
}
}
val passwordCookie = context.cookieStore.cookies.first { it.name.equals("vg_password") }.let { cookie ->
BasicClientCookie(cookie.name, cookie.value).apply {
domain = "viper.click"
}
}
listOf(useridCookie, passwordCookie)
} else {
emptyList()
}
}
private var authenticated = false
init {
context.cookieStore = BasicCookieStore()
fun init() {
coroutineScope.launch {
eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collect {
authenticate(it.settings)
}
}
authenticate(settingsService.settings)
}
private fun authenticate(settings: Settings) {
authenticated = false
if (!settings.viperSettings.login) {
log.debug("Authentication option is disabled")
context.cookieStore.clear()
authenticated = false
loggedUser = ""
synchronized(vgCookies) {
vgCookies.clear()
}
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
return
}
@@ -75,8 +56,11 @@ internal class VGAuthService(
val password = settings.viperSettings.password
if (username.isEmpty() || password.isEmpty()) {
log.error("Cannot authenticate with ViperGirls credentials, username or password is empty")
context.cookieStore.clear()
authenticated = false
loggedUser = ""
synchronized(vgCookies) {
vgCookies.clear()
}
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
return
}
@@ -90,8 +74,12 @@ internal class VGAuthService(
)
)
}
log.info("Authenticating: ${postAuth.uri}")
try {
val context = HttpClientContext.create().apply {
cookieStore =
BasicCookieStore()
}
cm.client.execute(postAuth, context) { response ->
if (response.code / 100 != 2) {
throw VripperException("Unexpected response code returned ${response.code}")
@@ -99,21 +87,34 @@ internal class VGAuthService(
val responseBody = EntityUtils.toString(response.entity)
log.debug("Authentication with ViperGirls response body:{}", responseBody)
}
if (context.cookieStore.cookies.stream().map { obj: Cookie -> obj.name }
.noneMatch { e: String -> e == "vg_userid" }) {
val userIdCookie = context.cookieStore.cookies.find { it.name == "vg_userid" }
val passwordCookie = context.cookieStore.cookies.find { it.name == "vg_password" }
if (userIdCookie == null || passwordCookie == null) {
log.error(
"Failed to authenticate user with {}, missing vg_userid cookie",
"Failed to authenticate user with {}, missing vg_userid/vg_password cookie",
settings.viperSettings.host
)
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
return
}
synchronized(vgCookies) {
vgCookies.clear()
vgCookies.add(BasicClientCookie(userIdCookie.name, userIdCookie.value).apply {
domain = userIdCookie.domain
})
vgCookies.add(BasicClientCookie(passwordCookie.name, passwordCookie.value).apply {
domain = passwordCookie.domain
})
}
} catch (e: Exception) {
context.cookieStore.clear()
loggedUser = ""
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
log.error(
"Failed to authenticate user with " + settings.viperSettings.host, e
)
authenticated = false
loggedUser = ""
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
return
}
authenticated = true
@@ -122,8 +123,35 @@ internal class VGAuthService(
}
fun leaveThanks(postEntity: PostEntity) {
val context = createVgContext()
taskRunner.submit(
LeaveThanksTask(postEntity, authenticated, context)
)
}
fun createVgContext(): HttpClientContext {
val context = HttpClientContext().apply { cookieStore = BasicCookieStore() }
synchronized(vgCookies) {
if (authenticated && vgCookies.isNotEmpty()) {
vgCookies.forEach { c -> context.cookieStore.addCookie(c) }
}
}
return context
}
fun createClickContext(): HttpClientContext {
val context = HttpClientContext().apply { cookieStore = BasicCookieStore() }
synchronized(vgCookies) {
if (authenticated && vgCookies.isNotEmpty()) {
vgCookies.forEach { c ->
context.cookieStore.addCookie(
BasicClientCookie(
c.name,
c.value
).apply { domain = "viper.click" })
}
}
}
return context
}
}
@@ -1,11 +1,7 @@
package me.vripper.tasks
import me.vripper.download.DownloadService
import me.vripper.model.ThreadPostId
import me.vripper.services.DataTransaction
import me.vripper.services.MetadataService
import me.vripper.services.SettingsService
import me.vripper.services.ThreadCacheService
import me.vripper.services.*
import me.vripper.utilities.LoggerDelegate
import me.vripper.vgapi.PostItem
import me.vripper.vgapi.PostLookupAPIParser
@@ -45,7 +45,7 @@ internal class FetchMetadataTask(
RequestLimit.getPermit(1)
log.debug("Requesting {}", httpGet.uri)
val response = httpService.client.execute(httpGet, vgAuthService.context) {
val response = httpService.client.execute(httpGet, vgAuthService.createVgContext()) {
if (it.code / 100 != 2) {
throw DownloadException("Unexpected response code '${it.code}' for $httpGet")
}
@@ -0,0 +1,10 @@
package me.vripper.utilities
import java.security.MessageDigest
import javax.xml.bind.DatatypeConverter
fun md5Hex(data: String): String {
val md: MessageDigest = MessageDigest.getInstance("MD5")
md.update(data.toByteArray())
return DatatypeConverter.printHexBinary(md.digest()).lowercase()
}
@@ -11,7 +11,6 @@ import me.vripper.tasks.Tasks
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.RequestLimit
import org.apache.hc.client5.http.classic.methods.HttpGet
import org.apache.hc.client5.http.protocol.HttpClientContext
import org.apache.hc.core5.http.io.entity.EntityUtils
import org.apache.hc.core5.net.URIBuilder
import org.koin.core.component.KoinComponent
@@ -46,8 +45,7 @@ internal class PostLookupAPIParser(private val threadId: Long, private val postI
log.info("Requesting {}", httpGet.uri)
httpService.client.execute(
httpGet,
HttpClientContext.create()
.apply { vgAuthService.clickCookies.forEach { cookieStore.addCookie(it) } }
vgAuthService.createClickContext()
) { response ->
if (response.code / 100 != 2) {
throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
@@ -11,8 +11,6 @@ import me.vripper.tasks.Tasks
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.RequestLimit
import org.apache.hc.client5.http.classic.methods.HttpGet
import org.apache.hc.client5.http.cookie.BasicCookieStore
import org.apache.hc.client5.http.protocol.HttpClientContext
import org.apache.hc.core5.http.io.entity.EntityUtils
import org.apache.hc.core5.net.URIBuilder
import org.koin.core.component.KoinComponent
@@ -48,10 +46,7 @@ internal class ThreadLookupAPIParser(private val threadId: Long) : KoinComponent
RequestLimit.getPermit(1)
log.info("Requesting {}", httpGet.uri)
cm.client.execute(
httpGet, HttpClientContext.create().apply {
cookieStore = BasicCookieStore()
vgAuthService.clickCookies.forEach { cookieStore.addCookie(it) }
}
httpGet, vgAuthService.createClickContext()
) { response ->
if (response.code / 100 != 2) {
throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
@@ -24,31 +24,55 @@ class AboutFragment : Fragment("About") {
spacing = 15.0
imageview("icons/64x64.png")
vbox(spacing = 5.0) {
text("VRipper") {
text("VRipper ${ApplicationProperties.VERSION}") {
style {
fontWeight = FontWeight.BOLD
fontSize = Dimension(18.0, Dimension.LinearUnits.px)
}
}
text("Version ${ApplicationProperties.VERSION}")
text("Developed by dev-claw and VRipper working group")
hbox(spacing = 5.0) {
hyperlink {
imageview("icons/github-mark.png").apply {
isPreserveRatio = true
fitHeight = 32.0
if (widgetsController.currentSettings.darkMode) {
imageview("icons/github-mark-white.png").apply {
isPreserveRatio = true
fitHeight = 32.0
}
} else {
imageview("icons/github-mark.png").apply {
isPreserveRatio = true
fitHeight = 32.0
}
}
action {
openLink("https://github.com/dev-claw/vripper-project")
}
}
hyperlink {
imageview("icons/buymeacoffee-logo.png").apply {
isPreserveRatio = true
fitHeight = 32.0
}
form {
fieldset("Donation") {
minWidth = 400.0
field {
hyperlink {
imageview("icons/buymeacoffee-logo.png").apply {
isPreserveRatio = true
fitHeight = 32.0
}
action {
openLink("https://buymeacoffee.com/devclaw")
}
}
}
action {
openLink("https://buymeacoffee.com/devclaw")
field("ETH:") {
textfield("0xDdac82B16dC5E3D742fc915ffF583D8548A301cA") {
isEditable = false
}
}
field("BTC:") {
textfield("bc1qcqudnkrndwyadsjwrxww42svkf8trnzx3c8vlr") {
isEditable = false
}
}
}
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 4.7 KiB

@@ -1,3 +1,3 @@
spring.liquibase.enabled=false
grpc.enabled=false
grpc.enabled=true
grpc.port=30000