mirror of
https://github.com/dev-claw/vripper-project.git
synced 2026-08-19 08:35:41 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4a001efa9f | ||
|
|
7035b1cce0 | ||
|
|
b9862a5042 | ||
|
|
8bcdb80149 | ||
|
|
e44220321e |
@@ -20,9 +20,9 @@ Direct access to `vipergirls.to` domain.
|
||||
|
||||
## Installing VRipper
|
||||
|
||||
<img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/windows_logo.svg" width="100%" height="75"> Windows | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/linux_logo.svg" width="100%" height="75"> Linux | <img src="https://images.vexels.com/media/users/3/166401/isolated/preview/b82aa7ac3f736dd78570dd3fa3fa9e24-java-programming-language-icon-by-vexels.png" width="100%" height="75"> Java
|
||||
:---:|:---:|:---:|:---:
|
||||
[Installer (EXE)](https://github.com/death-claw/vripper-project/releases/download/6.4.0/vripper-windows-installer-6.4.0.exe) <br /> [Installer (MSI)](https://github.com/death-claw/vripper-project/releases/download/6.4.0/vripper-windows-installer-6.4.0.msi) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.4.0/vripper-windows-portable-6.4.0.zip) | [Installer (DMG)](https://github.com/death-claw/vripper-project/releases/download/6.4.0/vripper-macos-6.4.0.dmg) <br /> [Installer (PKG)](https://github.com/death-claw/vripper-project/releases/download/6.4.0/vripper-macos-6.4.0.pkg) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.4.0/vripper-macos-portable-6.4.0.zip) | [Linux (amd64) (DEB)](https://github.com/death-claw/vripper-project/releases/download/6.4.0/vripper-linux-6.4.0_amd64.deb) <br /> [Linux (x86_64) (RPM)](https://github.com/death-claw/vripper-project/releases/download/6.4.0/vripper-linux-6.4.0.x86_64.rpm) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.4.0/vripper-linux-portable-6.4.0.zip) | [Java GUI (noarch)](https://github.com/death-claw/vripper-project/releases/download/6.4.0/vripper-noarch-gui-6.4.0.jar) <br /> [Java Web (noarch)](https://github.com/death-claw/vripper-project/releases/download/6.4.0/vripper-noarch-web-6.4.0.jar)
|
||||
<img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/windows_logo.svg" width="100%" height="75"> Windows | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS (Intel) | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS (Apple silicon) | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/linux_logo.svg" width="100%" height="75"> Linux | <img src="https://images.vexels.com/media/users/3/166401/isolated/preview/b82aa7ac3f736dd78570dd3fa3fa9e24-java-programming-language-icon-by-vexels.png" width="100%" height="75"> Java
|
||||
:---:|:---:|:---:|:---:|:---:
|
||||
[Installer (EXE)](https://github.com/death-claw/vripper-project/releases/download/6.5.1/vripper-windows-installer-6.5.1.exe) <br /> [Installer (MSI)](https://github.com/death-claw/vripper-project/releases/download/6.5.1/vripper-windows-installer-6.5.1.msi) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.5.1/vripper-windows-portable-6.5.1.zip) | [Installer (DMG)](https://github.com/death-claw/vripper-project/releases/download/6.5.1/vripper-macos-6.5.1.x86_64.dmg) <br /> [Installer (PKG)](https://github.com/death-claw/vripper-project/releases/download/6.5.1/vripper-macos-6.5.1.x86_64.pkg) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.5.1/vripper-macos-portable-6.5.1.x86_64.zip) | [Installer (DMG)](https://github.com/death-claw/vripper-project/releases/download/6.5.1/vripper-macos-6.5.1.arm64.dmg) <br /> [Installer (PKG)](https://github.com/death-claw/vripper-project/releases/download/6.5.1/vripper-macos-6.5.1.arm64.pkg) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.5.1/vripper-macos-portable-6.5.1.arm64.zip) | [Linux (amd64) (DEB)](https://github.com/death-claw/vripper-project/releases/download/6.5.1/vripper-linux-6.5.1_amd64.deb) <br /> [Linux (x86_64) (RPM)](https://github.com/death-claw/vripper-project/releases/download/6.5.1/vripper-linux-6.5.1.x86_64.rpm) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/6.5.1/vripper-linux-portable-6.5.1.zip) | [Java GUI (noarch)](https://github.com/death-claw/vripper-project/releases/download/6.5.1/vripper-noarch-gui-6.5.1.jar) <br /> [Java Web (noarch)](https://github.com/death-claw/vripper-project/releases/download/6.5.1/vripper-noarch-web-6.5.1.jar)
|
||||
|
||||
Source code and previous versions are available on the [Releases page](https://github.com/death-claw/vripper-project/releases).
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
<maven.compiler.source>${java.version}</maven.compiler.source>
|
||||
<maven.compiler.target>${java.version}</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<revision>6.4.0</revision>
|
||||
<revision>6.5.1</revision>
|
||||
<kotlin.version>2.0.21</kotlin.version>
|
||||
<slf4j.version>2.0.16</slf4j.version>
|
||||
<logback.version>1.5.12</logback.version>
|
||||
@@ -31,13 +31,13 @@
|
||||
<kotlinx-serialization-json.version>1.7.3</kotlinx-serialization-json.version>
|
||||
<httpclient5.version>5.4</httpclient5.version>
|
||||
<httpcore5.version>5.3</httpcore5.version>
|
||||
<commons-codec.version>1.17.1</commons-codec.version>
|
||||
<liquibase-core.version>4.29.2</liquibase-core.version>
|
||||
<liquibase-slf4j.version>5.0.0</liquibase-slf4j.version>
|
||||
<grpc.kotlin.version>1.4.1</grpc.kotlin.version>
|
||||
<java.grpc.version>1.68.1</java.grpc.version>
|
||||
<protobuf.version>3.25.5</protobuf.version>
|
||||
<sqlite-jdbc.version>3.47.0.0</sqlite-jdbc.version>
|
||||
<bucket4j_jdk17-core.version>8.14.0</bucket4j_jdk17-core.version>
|
||||
</properties>
|
||||
<build>
|
||||
<plugins>
|
||||
@@ -186,11 +186,6 @@
|
||||
<groupId>org.apache.httpcomponents.core5</groupId>
|
||||
<version>${httpcore5.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
<groupId>commons-codec</groupId>
|
||||
<version>${commons-codec.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<artifactId>liquibase-core</artifactId>
|
||||
<groupId>org.liquibase</groupId>
|
||||
@@ -221,6 +216,11 @@
|
||||
<artifactId>protobuf-kotlin</artifactId>
|
||||
<version>${protobuf.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.bucket4j</groupId>
|
||||
<artifactId>bucket4j_jdk17-core</artifactId>
|
||||
<version>${bucket4j_jdk17-core.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
</project>
|
||||
|
||||
@@ -75,10 +75,6 @@
|
||||
<artifactId>caffeine</artifactId>
|
||||
<groupId>com.github.ben-manes.caffeine</groupId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
<groupId>commons-codec</groupId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>net.java.dev.jna</groupId>
|
||||
<artifactId>jna-platform</artifactId>
|
||||
@@ -111,7 +107,6 @@
|
||||
<dependency>
|
||||
<groupId>com.bucket4j</groupId>
|
||||
<artifactId>bucket4j_jdk17-core</artifactId>
|
||||
<version>8.14.0</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
@@ -8,7 +8,6 @@ import me.vripper.data.repositories.impl.ImageRepositoryImpl
|
||||
import me.vripper.data.repositories.impl.MetadataRepositoryImpl
|
||||
import me.vripper.data.repositories.impl.PostRepositoryImpl
|
||||
import me.vripper.data.repositories.impl.ThreadRepositoryImpl
|
||||
import me.vripper.download.DownloadService
|
||||
import me.vripper.event.EventBus
|
||||
import me.vripper.host.*
|
||||
import me.vripper.services.*
|
||||
@@ -45,16 +44,16 @@ val coreModule = module {
|
||||
RetryPolicyService(get(), get())
|
||||
}
|
||||
single<HTTPService> {
|
||||
HTTPService(get(), get())
|
||||
HTTPService(get())
|
||||
}
|
||||
single<VGAuthService> {
|
||||
VGAuthService(get(), get(), get())
|
||||
VGAuthService(get(), get())
|
||||
}
|
||||
single<ThreadCacheService> {
|
||||
ThreadCacheService(get())
|
||||
ThreadCacheService(get(), get())
|
||||
}
|
||||
single<DownloadService> {
|
||||
DownloadService(get(), get(), get(), get(), get())
|
||||
DownloadService(get(), get(), get(), get())
|
||||
}
|
||||
single<DownloadSpeedService> {
|
||||
DownloadSpeedService(get())
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
package me.vripper.download
|
||||
|
||||
import kotlinx.coroutines.*
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.model.Settings
|
||||
import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase
|
||||
import org.apache.hc.client5.http.cookie.BasicCookieStore
|
||||
import org.apache.hc.client5.http.protocol.HttpClientContext
|
||||
import org.koin.core.component.KoinComponent
|
||||
|
||||
internal class ImageDownloadContext(val imageEntity: ImageEntity, val settings: Settings) : KoinComponent {
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||
private val jobs = mutableListOf<Job>()
|
||||
val httpContext: HttpClientContext =
|
||||
HttpClientContext.create().apply { cookieStore = BasicCookieStore() }
|
||||
val requests = mutableListOf<HttpUriRequestBase>()
|
||||
val postId = imageEntity.postIdRef
|
||||
|
||||
fun cancelCoroutines() {
|
||||
runBlocking {
|
||||
coroutineScope.cancel()
|
||||
jobs.forEach { job -> job.cancelAndJoin() }
|
||||
}
|
||||
}
|
||||
|
||||
fun launchCoroutine(block: suspend CoroutineScope.() -> Unit): Job {
|
||||
return coroutineScope.launch(block = block).also { job -> jobs.add(job) }
|
||||
}
|
||||
}
|
||||
@@ -1,154 +0,0 @@
|
||||
package me.vripper.download
|
||||
|
||||
import dev.failsafe.function.CheckedRunnable
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.entities.Status
|
||||
import me.vripper.exception.DownloadException
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.host.DownloadedImage
|
||||
import me.vripper.host.Host
|
||||
import me.vripper.host.ImageMimeType
|
||||
import me.vripper.model.Settings
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.VGAuthService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.PathUtils.getExtension
|
||||
import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
|
||||
import me.vripper.utilities.PathUtils.sanitize
|
||||
import org.koin.core.component.KoinComponent
|
||||
import org.koin.core.component.inject
|
||||
import java.io.IOException
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.StandardCopyOption
|
||||
import java.util.*
|
||||
import kotlin.io.path.Path
|
||||
import kotlin.io.path.pathString
|
||||
|
||||
internal class ImageDownloadRunnable(
|
||||
val imageEntity: ImageEntity, val postRank: Int, private val settings: Settings
|
||||
) : KoinComponent, CheckedRunnable {
|
||||
private val log by LoggerDelegate()
|
||||
private val dataTransaction: DataTransaction by inject()
|
||||
private val vgauthService: VGAuthService by inject()
|
||||
private val hosts: List<Host> = getKoin().getAll()
|
||||
var completed = false
|
||||
var stopped = false
|
||||
|
||||
private lateinit var context: ImageDownloadContext
|
||||
|
||||
fun download() {
|
||||
try {
|
||||
imageEntity.status = Status.DOWNLOADING
|
||||
imageEntity.downloaded = 0
|
||||
dataTransaction.updateImage(imageEntity)
|
||||
synchronized(imageEntity.postId.toString().intern()) {
|
||||
val post = dataTransaction.findPostById(context.postId)
|
||||
if (post.status != Status.DOWNLOADING) {
|
||||
post.status = Status.DOWNLOADING
|
||||
dataTransaction.updatePost(post)
|
||||
vgauthService.leaveThanks(post)
|
||||
}
|
||||
}
|
||||
log.debug("Getting image url and name from ${imageEntity.url} using ${imageEntity.host}")
|
||||
val host = hosts.first { it.isSupported(imageEntity.url) }
|
||||
val downloadedImage = host.downloadInternal(imageEntity.url, context)
|
||||
log.debug("Resolved name for ${imageEntity.url}: ${downloadedImage.name}")
|
||||
log.debug("Downloaded image {} to {}", imageEntity.url, downloadedImage.path)
|
||||
synchronized(imageEntity.postId.toString().intern()) {
|
||||
val post = dataTransaction.findPostById(context.postId)
|
||||
val downloadDirectory = Path(post.downloadDirectory, post.folderName).pathString
|
||||
checkImageTypeAndRename(
|
||||
downloadDirectory, downloadedImage, imageEntity.index
|
||||
)
|
||||
if (imageEntity.downloaded == imageEntity.size && imageEntity.size > 0) {
|
||||
imageEntity.status = Status.FINISHED
|
||||
post.done += 1
|
||||
post.downloaded += imageEntity.size
|
||||
dataTransaction.updatePost(post)
|
||||
} else {
|
||||
imageEntity.status = Status.ERROR
|
||||
}
|
||||
dataTransaction.updateImage(imageEntity)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
if (stopped) {
|
||||
return
|
||||
}
|
||||
imageEntity.status = Status.ERROR
|
||||
dataTransaction.updateImage(imageEntity)
|
||||
throw DownloadException(e)
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(HostException::class)
|
||||
private fun checkImageTypeAndRename(
|
||||
downloadDirectory: String, downloadedImage: DownloadedImage, index: Int
|
||||
) {
|
||||
val existingExtension = getExtension(downloadedImage.name).lowercase()
|
||||
val fileNameWithoutExtension = getFileNameWithoutExtension(downloadedImage.name)
|
||||
val extension = when (downloadedImage.type) {
|
||||
ImageMimeType.IMAGE_BMP -> "BMP"
|
||||
ImageMimeType.IMAGE_GIF -> "GIF"
|
||||
ImageMimeType.IMAGE_JPEG -> "JPG"
|
||||
ImageMimeType.IMAGE_PNG -> "PNG"
|
||||
ImageMimeType.IMAGE_WEBP -> "WEBP"
|
||||
}
|
||||
val filename =
|
||||
if (existingExtension.isBlank()) "${sanitize(downloadedImage.name)}.$extension" else "${
|
||||
sanitize(
|
||||
fileNameWithoutExtension
|
||||
)
|
||||
}.$extension"
|
||||
try {
|
||||
val downloadDestinationFolder = Path.of(downloadDirectory)
|
||||
Files.createDirectories(downloadDestinationFolder)
|
||||
val finalFilename = "${
|
||||
if (settings.downloadSettings.forceOrder) String.format(
|
||||
"%03d_", index + 1
|
||||
) else ""
|
||||
}$filename"
|
||||
imageEntity.filename = finalFilename
|
||||
val imageDownloadPath = downloadDestinationFolder.resolve(finalFilename)
|
||||
Files.copy(downloadedImage.path, imageDownloadPath, StandardCopyOption.REPLACE_EXISTING)
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Failed to rename the image", e)
|
||||
} finally {
|
||||
try {
|
||||
Files.delete(downloadedImage.path)
|
||||
} catch (_: IOException) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun run() {
|
||||
context = ImageDownloadContext(imageEntity, settings)
|
||||
try {
|
||||
if (stopped) {
|
||||
return
|
||||
}
|
||||
download()
|
||||
} finally {
|
||||
completed = true
|
||||
context.cancelCoroutines()
|
||||
}
|
||||
}
|
||||
|
||||
fun stop() {
|
||||
stopped = true
|
||||
context.requests.forEach { it.abort() }
|
||||
context.cancelCoroutines()
|
||||
dataTransaction.updateImage(context.imageEntity)
|
||||
}
|
||||
|
||||
override fun equals(other: Any?): Boolean {
|
||||
if (this === other) return true
|
||||
if (other == null || javaClass != other.javaClass) return false
|
||||
val that = other as ImageDownloadRunnable
|
||||
return imageEntity.id == that.imageEntity.id
|
||||
}
|
||||
|
||||
override fun hashCode(): Int {
|
||||
return Objects.hash(imageEntity.id)
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,9 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.download.ImageDownloadContext
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.HtmlUtils
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.download.ImageDownloadContext
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.*
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
|
||||
@@ -4,10 +4,10 @@ import kotlinx.coroutines.cancelAndJoin
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import me.vripper.download.ImageDownloadContext
|
||||
import me.vripper.exception.DownloadException
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.HtmlUtils
|
||||
@@ -22,7 +22,6 @@ import org.w3c.dom.Document
|
||||
import java.io.BufferedOutputStream
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import kotlin.Throws
|
||||
|
||||
internal abstract class Host(
|
||||
val hostName: String,
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.download.ImageDownloadContext
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.*
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.HtmlUtils
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.download.ImageDownloadContext
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.*
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.download.ImageDownloadContext
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.*
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.HtmlUtils
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.download.ImageDownloadContext
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.*
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.download.ImageDownloadContext
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.*
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.download.ImageDownloadContext
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.*
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.download.ImageDownloadContext
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.HtmlProcessorException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.*
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.HtmlUtils
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.download.ImageDownloadContext
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.*
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.HtmlUtils
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.download.ImageDownloadContext
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.*
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.download.ImageDownloadContext
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.*
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.download.ImageDownloadContext
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.*
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.download.ImageDownloadContext
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.*
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.download.ImageDownloadContext
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.*
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.download.ImageDownloadContext
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.*
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
|
||||
@@ -1,17 +1,30 @@
|
||||
package me.vripper.listeners
|
||||
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.MetadataService
|
||||
import me.vripper.services.*
|
||||
import org.koin.core.component.KoinComponent
|
||||
import org.koin.core.component.inject
|
||||
|
||||
open class OnStartupListener : KoinComponent {
|
||||
private val dataTransaction: DataTransaction by inject()
|
||||
private val metadataService: MetadataService by inject()
|
||||
private val settingsService: SettingsService by inject()
|
||||
private val vgAuthService: VGAuthService by inject()
|
||||
private val downloadSpeedService: DownloadSpeedService by inject()
|
||||
private val httpService: HTTPService by inject()
|
||||
private val retryPolicyService: RetryPolicyService by inject()
|
||||
private val threadCacheService: ThreadCacheService by inject()
|
||||
private val downloadService: DownloadService by inject()
|
||||
|
||||
open fun run() {
|
||||
dataTransaction.setDownloadingToStopped()
|
||||
dataTransaction.stopImagesByPostIdAndIsNotCompleted()
|
||||
vgAuthService.init()
|
||||
httpService.init()
|
||||
retryPolicyService.init()
|
||||
threadCacheService.init()
|
||||
settingsService.init()
|
||||
metadataService.init()
|
||||
downloadSpeedService.init()
|
||||
downloadService.init()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ data class ViperSettings(
|
||||
val password: String = "",
|
||||
val thanks: Boolean = false,
|
||||
val host: String = "https://vipergirls.to",
|
||||
val requestLimit: Long = 4,
|
||||
val requestLimit: Long = 2,
|
||||
val fetchMetadata: Boolean = false,
|
||||
)
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@ package me.vripper.services
|
||||
import kotlinx.coroutines.flow.*
|
||||
import kotlinx.coroutines.time.sample
|
||||
import kotlinx.serialization.json.Json
|
||||
import me.vripper.download.DownloadService
|
||||
import me.vripper.entities.*
|
||||
import me.vripper.event.*
|
||||
import me.vripper.exception.PostParseException
|
||||
@@ -119,6 +118,9 @@ internal class AppEndpointService(
|
||||
)
|
||||
}
|
||||
val threadLookupResult = threadCacheService[thread.threadId]
|
||||
if (threadLookupResult.error.isNotBlank()) {
|
||||
throw PostParseException("Error occurred for threadId = $threadId, ${threadLookupResult.error}")
|
||||
}
|
||||
threadLookupResult.postItemList.map { postItem ->
|
||||
PostSelection(
|
||||
postItem.threadId,
|
||||
@@ -132,23 +134,13 @@ internal class AppEndpointService(
|
||||
postItem.forum,
|
||||
postItem.imageItemList.map { it.thumbLink })
|
||||
}.ifEmpty {
|
||||
log.error(
|
||||
String.format(
|
||||
"Failed to get links for threadId = %s", threadId
|
||||
)
|
||||
)
|
||||
throw PostParseException(
|
||||
String.format(
|
||||
"Failed to get links for threadId = %s", threadId
|
||||
)
|
||||
"Nothing found for threadId = $threadId"
|
||||
)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
throw PostParseException(
|
||||
String.format(
|
||||
"Failed to get links for threadId = %s, %s", threadId, e.message
|
||||
)
|
||||
)
|
||||
log.error(e.message)
|
||||
throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+174
-10
@@ -1,8 +1,9 @@
|
||||
package me.vripper.download
|
||||
package me.vripper.services
|
||||
|
||||
import dev.failsafe.Failsafe
|
||||
import dev.failsafe.RetryPolicy
|
||||
import kotlinx.coroutines.Runnable
|
||||
import dev.failsafe.function.CheckedRunnable
|
||||
import kotlinx.coroutines.*
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.entities.PostEntity
|
||||
import me.vripper.entities.Status
|
||||
@@ -10,24 +11,39 @@ import me.vripper.event.ErrorCountEvent
|
||||
import me.vripper.event.EventBus
|
||||
import me.vripper.event.QueueStateEvent
|
||||
import me.vripper.event.StoppedEvent
|
||||
import me.vripper.exception.DownloadException
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.host.DownloadedImage
|
||||
import me.vripper.host.Host
|
||||
import me.vripper.host.ImageMimeType
|
||||
import me.vripper.model.ErrorCount
|
||||
import me.vripper.model.QueueState
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.RetryPolicyService
|
||||
import me.vripper.services.SettingsService
|
||||
import me.vripper.services.VGAuthService
|
||||
import me.vripper.model.Settings
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.PathUtils.getExtension
|
||||
import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
|
||||
import me.vripper.utilities.PathUtils.sanitize
|
||||
import me.vripper.utilities.downloadRunner
|
||||
import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase
|
||||
import org.apache.hc.client5.http.cookie.BasicCookieStore
|
||||
import org.apache.hc.client5.http.protocol.HttpClientContext
|
||||
import org.jetbrains.exposed.sql.transactions.transaction
|
||||
import org.koin.core.component.KoinComponent
|
||||
import org.koin.core.component.inject
|
||||
import java.io.IOException
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.StandardCopyOption
|
||||
import java.util.*
|
||||
import java.util.concurrent.locks.ReentrantLock
|
||||
import kotlin.concurrent.withLock
|
||||
import kotlin.io.path.Path
|
||||
import kotlin.io.path.pathString
|
||||
|
||||
internal class DownloadService(
|
||||
private val settingsService: SettingsService,
|
||||
private val dataTransaction: DataTransaction,
|
||||
private val retryPolicyService: RetryPolicyService,
|
||||
private val vgAuthService: VGAuthService,
|
||||
private val eventBus: EventBus
|
||||
) {
|
||||
private val maxPoolSize: Int = 24
|
||||
@@ -37,7 +53,155 @@ internal class DownloadService(
|
||||
private val lock = ReentrantLock()
|
||||
private val condition = lock.newCondition()
|
||||
|
||||
init {
|
||||
internal class ImageDownloadContext(val imageEntity: ImageEntity, val settings: Settings) : KoinComponent {
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||
private val jobs = mutableListOf<Job>()
|
||||
val httpContext: HttpClientContext =
|
||||
HttpClientContext.create().apply { cookieStore = BasicCookieStore() }
|
||||
val requests = mutableListOf<HttpUriRequestBase>()
|
||||
val postId = imageEntity.postIdRef
|
||||
|
||||
fun cancelCoroutines() {
|
||||
runBlocking {
|
||||
coroutineScope.cancel()
|
||||
jobs.forEach { job -> job.cancelAndJoin() }
|
||||
}
|
||||
}
|
||||
|
||||
fun launchCoroutine(block: suspend CoroutineScope.() -> Unit): Job {
|
||||
return coroutineScope.launch(block = block).also { job -> jobs.add(job) }
|
||||
}
|
||||
}
|
||||
|
||||
internal class ImageDownloadRunnable(
|
||||
val imageEntity: ImageEntity, val postRank: Int, private val settings: Settings
|
||||
) : KoinComponent, CheckedRunnable {
|
||||
private val log by LoggerDelegate()
|
||||
private val dataTransaction: DataTransaction by inject()
|
||||
private val vgauthService: VGAuthService by inject()
|
||||
private val hosts: List<Host> = getKoin().getAll()
|
||||
var completed = false
|
||||
var stopped = false
|
||||
|
||||
private lateinit var context: ImageDownloadContext
|
||||
|
||||
fun download() {
|
||||
try {
|
||||
imageEntity.status = Status.DOWNLOADING
|
||||
imageEntity.downloaded = 0
|
||||
dataTransaction.updateImage(imageEntity)
|
||||
synchronized(imageEntity.postId.toString().intern()) {
|
||||
val post = dataTransaction.findPostById(context.postId)
|
||||
if (post.status != Status.DOWNLOADING) {
|
||||
post.status = Status.DOWNLOADING
|
||||
dataTransaction.updatePost(post)
|
||||
vgauthService.leaveThanks(post)
|
||||
}
|
||||
}
|
||||
log.debug("Getting image url and name from ${imageEntity.url} using ${imageEntity.host}")
|
||||
val host = hosts.first { it.isSupported(imageEntity.url) }
|
||||
val downloadedImage = host.downloadInternal(imageEntity.url, context)
|
||||
log.debug("Resolved name for ${imageEntity.url}: ${downloadedImage.name}")
|
||||
log.debug("Downloaded image {} to {}", imageEntity.url, downloadedImage.path)
|
||||
synchronized(imageEntity.postId.toString().intern()) {
|
||||
val post = dataTransaction.findPostById(context.postId)
|
||||
val downloadDirectory = Path(post.downloadDirectory, post.folderName).pathString
|
||||
checkImageTypeAndRename(
|
||||
downloadDirectory, downloadedImage, imageEntity.index
|
||||
)
|
||||
if (imageEntity.downloaded == imageEntity.size && imageEntity.size > 0) {
|
||||
imageEntity.status = Status.FINISHED
|
||||
post.done += 1
|
||||
post.downloaded += imageEntity.size
|
||||
dataTransaction.updatePost(post)
|
||||
} else {
|
||||
imageEntity.status = Status.ERROR
|
||||
}
|
||||
dataTransaction.updateImage(imageEntity)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
if (stopped) {
|
||||
return
|
||||
}
|
||||
imageEntity.status = Status.ERROR
|
||||
dataTransaction.updateImage(imageEntity)
|
||||
throw DownloadException(e)
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(HostException::class)
|
||||
private fun checkImageTypeAndRename(
|
||||
downloadDirectory: String, downloadedImage: DownloadedImage, index: Int
|
||||
) {
|
||||
val existingExtension = getExtension(downloadedImage.name).lowercase()
|
||||
val fileNameWithoutExtension = getFileNameWithoutExtension(downloadedImage.name)
|
||||
val extension = when (downloadedImage.type) {
|
||||
ImageMimeType.IMAGE_BMP -> "BMP"
|
||||
ImageMimeType.IMAGE_GIF -> "GIF"
|
||||
ImageMimeType.IMAGE_JPEG -> "JPG"
|
||||
ImageMimeType.IMAGE_PNG -> "PNG"
|
||||
ImageMimeType.IMAGE_WEBP -> "WEBP"
|
||||
}
|
||||
val filename =
|
||||
if (existingExtension.isBlank()) "${sanitize(downloadedImage.name)}.$extension" else "${
|
||||
sanitize(
|
||||
fileNameWithoutExtension
|
||||
)
|
||||
}.$extension"
|
||||
try {
|
||||
val downloadDestinationFolder = Path.of(downloadDirectory)
|
||||
Files.createDirectories(downloadDestinationFolder)
|
||||
val finalFilename = "${
|
||||
if (settings.downloadSettings.forceOrder) String.format(
|
||||
"%03d_", index + 1
|
||||
) else ""
|
||||
}$filename"
|
||||
imageEntity.filename = finalFilename
|
||||
val imageDownloadPath = downloadDestinationFolder.resolve(finalFilename)
|
||||
Files.copy(downloadedImage.path, imageDownloadPath, StandardCopyOption.REPLACE_EXISTING)
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Failed to rename the image", e)
|
||||
} finally {
|
||||
try {
|
||||
Files.delete(downloadedImage.path)
|
||||
} catch (_: IOException) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun run() {
|
||||
context = ImageDownloadContext(imageEntity, settings)
|
||||
try {
|
||||
if (stopped) {
|
||||
return
|
||||
}
|
||||
download()
|
||||
} finally {
|
||||
completed = true
|
||||
context.cancelCoroutines()
|
||||
}
|
||||
}
|
||||
|
||||
fun stop() {
|
||||
stopped = true
|
||||
context.requests.forEach { it.abort() }
|
||||
context.cancelCoroutines()
|
||||
dataTransaction.updateImage(context.imageEntity)
|
||||
}
|
||||
|
||||
override fun equals(other: Any?): Boolean {
|
||||
if (this === other) return true
|
||||
if (other == null || javaClass != other.javaClass) return false
|
||||
val that = other as ImageDownloadRunnable
|
||||
return imageEntity.id == that.imageEntity.id
|
||||
}
|
||||
|
||||
override fun hashCode(): Int {
|
||||
return Objects.hash(imageEntity.id)
|
||||
}
|
||||
}
|
||||
|
||||
fun init() {
|
||||
Thread.ofVirtual().name("Download Loop").unstarted(Runnable {
|
||||
val accepted: MutableList<ImageDownloadRunnable> = mutableListOf()
|
||||
val candidates: MutableList<ImageDownloadRunnable> = mutableListOf()
|
||||
@@ -59,7 +223,7 @@ internal class DownloadService(
|
||||
candidates.clear()
|
||||
try {
|
||||
condition.await()
|
||||
} catch (e: InterruptedException) {
|
||||
} catch (_: InterruptedException) {
|
||||
Thread.currentThread().interrupt()
|
||||
}
|
||||
}
|
||||
@@ -189,7 +353,7 @@ internal class DownloadService(
|
||||
|
||||
private fun candidateCount(): Map<Byte, Int> {
|
||||
val map: MutableMap<Byte, Int> = mutableMapOf()
|
||||
Host.getHosts().values.forEach { host: Byte ->
|
||||
Host.Companion.getHosts().values.forEach { host: Byte ->
|
||||
val imageDownloadRunnableList: List<ImageDownloadRunnable> = running.computeIfAbsent(
|
||||
host
|
||||
) { mutableListOf() }
|
||||
@@ -20,7 +20,7 @@ internal class DownloadSpeedService(
|
||||
private val bytesCount = AtomicLong(0)
|
||||
private var job: Job? = null
|
||||
|
||||
init {
|
||||
fun init() {
|
||||
coroutineScope.launch {
|
||||
eventBus.events.filterIsInstance(QueueStateEvent::class).collect {
|
||||
if (it.queueState.running + it.queueState.remaining > 0) {
|
||||
|
||||
@@ -10,15 +10,15 @@ import org.apache.hc.client5.http.cookie.StandardCookieSpec
|
||||
import org.apache.hc.client5.http.impl.DefaultRedirectStrategy
|
||||
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient
|
||||
import org.apache.hc.client5.http.impl.classic.HttpClients
|
||||
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager
|
||||
import org.apache.hc.client5.http.impl.io.BasicHttpClientConnectionManager
|
||||
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder
|
||||
import org.apache.hc.client5.http.io.HttpClientConnectionManager
|
||||
import org.apache.hc.core5.pool.PoolConcurrencyPolicy
|
||||
import org.apache.hc.core5.pool.PoolReusePolicy
|
||||
import org.apache.hc.core5.util.Timeout
|
||||
|
||||
internal class HTTPService(
|
||||
private val eventBus: EventBus,
|
||||
settingsService: SettingsService
|
||||
private val eventBus: EventBus
|
||||
) {
|
||||
|
||||
companion object {
|
||||
@@ -28,23 +28,14 @@ internal class HTTPService(
|
||||
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||
|
||||
private lateinit var pcm: PoolingHttpClientConnectionManager
|
||||
private lateinit var rc: RequestConfig
|
||||
private lateinit var cc: ConnectionConfig
|
||||
lateinit var client: CloseableHttpClient
|
||||
private var connectionTimeout = settingsService.settings.connectionSettings.timeout
|
||||
private var pcm: HttpClientConnectionManager = BasicHttpClientConnectionManager()
|
||||
private var rc: RequestConfig = RequestConfig.DEFAULT
|
||||
private var cc: ConnectionConfig = ConnectionConfig.DEFAULT
|
||||
private var connectionTimeout = 30
|
||||
private var connectionExpiryJob: Job? = null
|
||||
var client: CloseableHttpClient = HttpClients.createDefault()
|
||||
|
||||
init {
|
||||
buildRequestConfig()
|
||||
buildConnectionConfig()
|
||||
buildConnectionPool()
|
||||
buildClientBuilder()
|
||||
coroutineScope.launch {
|
||||
while (isActive) {
|
||||
pcm.closeExpired()
|
||||
delay(60_000)
|
||||
}
|
||||
}
|
||||
fun init() {
|
||||
coroutineScope.launch {
|
||||
eventBus
|
||||
.events
|
||||
@@ -64,13 +55,21 @@ internal class HTTPService(
|
||||
}
|
||||
|
||||
private fun buildConnectionPool() {
|
||||
connectionExpiryJob?.cancel()
|
||||
pcm = PoolingHttpClientConnectionManagerBuilder.create()
|
||||
.setPoolConcurrencyPolicy(PoolConcurrencyPolicy.LAX)
|
||||
.setConnPoolPolicy(PoolReusePolicy.FIFO)
|
||||
.setDefaultConnectionConfig(cc)
|
||||
.setMaxConnTotal(Int.MAX_VALUE)
|
||||
.setMaxConnPerRoute(Int.MAX_VALUE)
|
||||
.build()
|
||||
.build().also {
|
||||
connectionExpiryJob = coroutineScope.launch {
|
||||
while (isActive) {
|
||||
it.closeExpired()
|
||||
delay(60_000)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun buildRequestConfig() {
|
||||
@@ -95,6 +94,5 @@ internal class HTTPService(
|
||||
.disableAutomaticRetries()
|
||||
.setDefaultRequestConfig(rc)
|
||||
.build()
|
||||
|
||||
}
|
||||
}
|
||||
@@ -18,7 +18,7 @@ internal class RetryPolicyService(
|
||||
private var maxAttempts: Int = settingsService.settings.connectionSettings.maxAttempts
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||
|
||||
init {
|
||||
fun init() {
|
||||
coroutineScope.launch {
|
||||
eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collect {
|
||||
if (maxAttempts != it.settings.connectionSettings.maxAttempts) {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package me.vripper.services
|
||||
|
||||
import kotlinx.serialization.ExperimentalSerializationApi
|
||||
import kotlinx.serialization.encodeToString
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.decodeFromStream
|
||||
@@ -9,7 +10,7 @@ import me.vripper.exception.ValidationException
|
||||
import me.vripper.model.Settings
|
||||
import me.vripper.utilities.ApplicationProperties.VRIPPER_DIR
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
import me.vripper.utilities.md5Hex
|
||||
import java.io.FileWriter
|
||||
import java.nio.file.*
|
||||
import kotlin.io.path.readText
|
||||
@@ -28,16 +29,13 @@ class SettingsService(private val eventBus: EventBus) {
|
||||
}
|
||||
var settings = Settings()
|
||||
|
||||
init {
|
||||
init()
|
||||
}
|
||||
|
||||
private fun init() {
|
||||
fun init() {
|
||||
loadViperProxies()
|
||||
restore()
|
||||
eventBus.publishEvent(SettingsUpdateEvent(settings))
|
||||
}
|
||||
|
||||
@OptIn(ExperimentalSerializationApi::class)
|
||||
private fun loadViperProxies() {
|
||||
try {
|
||||
SettingsService::class.java.getResourceAsStream("/proxies.json")?.use {
|
||||
@@ -84,7 +82,7 @@ class SettingsService(private val eventBus: EventBus) {
|
||||
check(settings)
|
||||
val viperSettings = if (settings.viperSettings.login) {
|
||||
if (this.settings.viperSettings.password != settings.viperSettings.password) {
|
||||
settings.viperSettings.copy(password = DigestUtils.md5Hex(settings.viperSettings.password))
|
||||
settings.viperSettings.copy(password = md5Hex(settings.viperSettings.password))
|
||||
} else {
|
||||
settings.viperSettings
|
||||
}
|
||||
@@ -137,7 +135,7 @@ class SettingsService(private val eventBus: EventBus) {
|
||||
fun check(settings: Settings) {
|
||||
val path: Path = try {
|
||||
Paths.get(settings.downloadSettings.downloadPath)
|
||||
} catch (e: InvalidPathException) {
|
||||
} catch (_: InvalidPathException) {
|
||||
throw ValidationException(
|
||||
String.format(
|
||||
"%s is invalid", settings.downloadSettings.downloadPath
|
||||
@@ -189,9 +187,9 @@ class SettingsService(private val eventBus: EventBus) {
|
||||
)
|
||||
}
|
||||
|
||||
if (settings.viperSettings.requestLimit < 1 || settings.viperSettings.requestLimit > 6) {
|
||||
if (settings.viperSettings.requestLimit < 1 || settings.viperSettings.requestLimit > 5) {
|
||||
throw ValidationException(
|
||||
"Invalid request rate limit, values must be in [1,6]"
|
||||
"Invalid request rate limit, values must be in [1,5]"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import me.vripper.vgapi.ThreadLookupAPIParser
|
||||
import java.util.concurrent.ExecutionException
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
internal class ThreadCacheService(val eventBus: EventBus) {
|
||||
internal class ThreadCacheService(val eventBus: EventBus, val dataTransaction: DataTransaction) {
|
||||
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||
|
||||
@@ -28,7 +28,13 @@ internal class ThreadCacheService(val eventBus: EventBus) {
|
||||
|
||||
private val cache: LoadingCache<Long, ThreadItem> =
|
||||
Caffeine.newBuilder().expireAfterWrite(20, TimeUnit.MINUTES).build { threadId ->
|
||||
ThreadLookupAPIParser(threadId).parse()
|
||||
val threadItem = ThreadLookupAPIParser(threadId).parse()
|
||||
dataTransaction.findThreadByThreadId(threadItem.threadId).ifPresent {
|
||||
if (threadItem.postItemList.isNotEmpty()) {
|
||||
dataTransaction.update(it.copy(total = threadItem.postItemList.size))
|
||||
}
|
||||
}
|
||||
threadItem
|
||||
}
|
||||
|
||||
@Throws(ExecutionException::class)
|
||||
|
||||
@@ -18,38 +18,37 @@ import org.apache.hc.client5.http.classic.methods.HttpPost
|
||||
import org.apache.hc.client5.http.cookie.BasicCookieStore
|
||||
import org.apache.hc.client5.http.cookie.Cookie
|
||||
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
|
||||
import org.apache.hc.client5.http.impl.cookie.BasicClientCookie
|
||||
import org.apache.hc.client5.http.protocol.HttpClientContext
|
||||
import org.apache.hc.core5.http.io.entity.EntityUtils
|
||||
import org.apache.hc.core5.http.message.BasicNameValuePair
|
||||
|
||||
internal class VGAuthService(
|
||||
private val cm: HTTPService,
|
||||
private val settingsService: SettingsService,
|
||||
private val eventBus: EventBus
|
||||
private val eventBus: EventBus,
|
||||
) {
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||
private val log by LoggerDelegate()
|
||||
val context: HttpClientContext = HttpClientContext.create()
|
||||
private val vgCookies: MutableList<Cookie> = mutableListOf()
|
||||
var loggedUser = ""
|
||||
|
||||
private var authenticated = false
|
||||
|
||||
init {
|
||||
context.cookieStore = BasicCookieStore()
|
||||
fun init() {
|
||||
coroutineScope.launch {
|
||||
eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collect {
|
||||
authenticate(it.settings)
|
||||
}
|
||||
}
|
||||
authenticate(settingsService.settings)
|
||||
}
|
||||
|
||||
private fun authenticate(settings: Settings) {
|
||||
authenticated = false
|
||||
if (!settings.viperSettings.login) {
|
||||
log.debug("Authentication option is disabled")
|
||||
context.cookieStore.clear()
|
||||
authenticated = false
|
||||
loggedUser = ""
|
||||
synchronized(vgCookies) {
|
||||
vgCookies.clear()
|
||||
}
|
||||
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
|
||||
return
|
||||
}
|
||||
@@ -57,8 +56,11 @@ internal class VGAuthService(
|
||||
val password = settings.viperSettings.password
|
||||
if (username.isEmpty() || password.isEmpty()) {
|
||||
log.error("Cannot authenticate with ViperGirls credentials, username or password is empty")
|
||||
context.cookieStore.clear()
|
||||
authenticated = false
|
||||
loggedUser = ""
|
||||
synchronized(vgCookies) {
|
||||
vgCookies.clear()
|
||||
}
|
||||
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
|
||||
return
|
||||
}
|
||||
@@ -72,8 +74,12 @@ internal class VGAuthService(
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
log.info("Authenticating: ${postAuth.uri}")
|
||||
try {
|
||||
val context = HttpClientContext.create().apply {
|
||||
cookieStore =
|
||||
BasicCookieStore()
|
||||
}
|
||||
cm.client.execute(postAuth, context) { response ->
|
||||
if (response.code / 100 != 2) {
|
||||
throw VripperException("Unexpected response code returned ${response.code}")
|
||||
@@ -81,21 +87,34 @@ internal class VGAuthService(
|
||||
val responseBody = EntityUtils.toString(response.entity)
|
||||
log.debug("Authentication with ViperGirls response body:{}", responseBody)
|
||||
}
|
||||
if (context.cookieStore.cookies.stream().map { obj: Cookie -> obj.name }
|
||||
.noneMatch { e: String -> e == "vg_userid" }) {
|
||||
|
||||
val userIdCookie = context.cookieStore.cookies.find { it.name == "vg_userid" }
|
||||
val passwordCookie = context.cookieStore.cookies.find { it.name == "vg_password" }
|
||||
|
||||
if (userIdCookie == null || passwordCookie == null) {
|
||||
log.error(
|
||||
"Failed to authenticate user with {}, missing vg_userid cookie",
|
||||
"Failed to authenticate user with {}, missing vg_userid/vg_password cookie",
|
||||
settings.viperSettings.host
|
||||
)
|
||||
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
|
||||
return
|
||||
}
|
||||
synchronized(vgCookies) {
|
||||
vgCookies.clear()
|
||||
vgCookies.add(BasicClientCookie(userIdCookie.name, userIdCookie.value).apply {
|
||||
domain = userIdCookie.domain
|
||||
})
|
||||
vgCookies.add(BasicClientCookie(passwordCookie.name, passwordCookie.value).apply {
|
||||
domain = passwordCookie.domain
|
||||
})
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
context.cookieStore.clear()
|
||||
loggedUser = ""
|
||||
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
|
||||
log.error(
|
||||
"Failed to authenticate user with " + settings.viperSettings.host, e
|
||||
)
|
||||
authenticated = false
|
||||
loggedUser = ""
|
||||
eventBus.publishEvent(VGUserLoginEvent(loggedUser))
|
||||
return
|
||||
}
|
||||
authenticated = true
|
||||
@@ -104,8 +123,35 @@ internal class VGAuthService(
|
||||
}
|
||||
|
||||
fun leaveThanks(postEntity: PostEntity) {
|
||||
val context = createVgContext()
|
||||
taskRunner.submit(
|
||||
LeaveThanksTask(postEntity, authenticated, context)
|
||||
)
|
||||
}
|
||||
|
||||
fun createVgContext(): HttpClientContext {
|
||||
val context = HttpClientContext().apply { cookieStore = BasicCookieStore() }
|
||||
synchronized(vgCookies) {
|
||||
if (authenticated && vgCookies.isNotEmpty()) {
|
||||
vgCookies.forEach { c -> context.cookieStore.addCookie(c) }
|
||||
}
|
||||
}
|
||||
return context
|
||||
}
|
||||
|
||||
fun createClickContext(): HttpClientContext {
|
||||
val context = HttpClientContext().apply { cookieStore = BasicCookieStore() }
|
||||
synchronized(vgCookies) {
|
||||
if (authenticated && vgCookies.isNotEmpty()) {
|
||||
vgCookies.forEach { c ->
|
||||
context.cookieStore.addCookie(
|
||||
BasicClientCookie(
|
||||
c.name,
|
||||
c.value
|
||||
).apply { domain = "viper.click" })
|
||||
}
|
||||
}
|
||||
}
|
||||
return context
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,7 @@
|
||||
package me.vripper.tasks
|
||||
|
||||
import me.vripper.download.DownloadService
|
||||
import me.vripper.model.ThreadPostId
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.MetadataService
|
||||
import me.vripper.services.SettingsService
|
||||
import me.vripper.services.ThreadCacheService
|
||||
import me.vripper.services.*
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.vgapi.PostItem
|
||||
import me.vripper.vgapi.PostLookupAPIParser
|
||||
|
||||
@@ -45,7 +45,7 @@ internal class FetchMetadataTask(
|
||||
|
||||
RequestLimit.getPermit(1)
|
||||
log.debug("Requesting {}", httpGet.uri)
|
||||
val response = httpService.client.execute(httpGet, vgAuthService.context) {
|
||||
val response = httpService.client.execute(httpGet, vgAuthService.createVgContext()) {
|
||||
if (it.code / 100 != 2) {
|
||||
throw DownloadException("Unexpected response code '${it.code}' for $httpGet")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
package me.vripper.utilities
|
||||
|
||||
import java.security.MessageDigest
|
||||
import javax.xml.bind.DatatypeConverter
|
||||
|
||||
fun md5Hex(data: String): String {
|
||||
val md: MessageDigest = MessageDigest.getInstance("MD5")
|
||||
md.update(data.toByteArray())
|
||||
return DatatypeConverter.printHexBinary(md.digest()).lowercase()
|
||||
}
|
||||
@@ -6,7 +6,6 @@ import me.vripper.exception.DownloadException
|
||||
import me.vripper.exception.PostParseException
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.services.RetryPolicyService
|
||||
import me.vripper.services.SettingsService
|
||||
import me.vripper.services.VGAuthService
|
||||
import me.vripper.tasks.Tasks
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
@@ -21,7 +20,6 @@ import javax.xml.parsers.SAXParserFactory
|
||||
|
||||
internal class PostLookupAPIParser(private val threadId: Long, private val postId: Long) : KoinComponent {
|
||||
private val log by LoggerDelegate()
|
||||
private val settingsService: SettingsService by inject()
|
||||
private val retryPolicyService: RetryPolicyService by inject()
|
||||
private val httpService: HTTPService by inject()
|
||||
private val vgAuthService: VGAuthService by inject()
|
||||
@@ -30,7 +28,7 @@ internal class PostLookupAPIParser(private val threadId: Long, private val postI
|
||||
fun parse(): ThreadItem? {
|
||||
log.debug("Parsing post $postId")
|
||||
val httpGet =
|
||||
HttpGet(URIBuilder("${settingsService.settings.viperSettings.host}/vr.php").also {
|
||||
HttpGet(URIBuilder("https://viper.click/vr.php").also {
|
||||
it.setParameter(
|
||||
"p", postId.toString()
|
||||
)
|
||||
@@ -44,9 +42,10 @@ internal class PostLookupAPIParser(private val threadId: Long, private val postI
|
||||
)
|
||||
}.get(CheckedSupplier {
|
||||
RequestLimit.getPermit(1)
|
||||
log.debug("Requesting {}", httpGet.uri)
|
||||
log.info("Requesting {}", httpGet.uri)
|
||||
httpService.client.execute(
|
||||
httpGet, vgAuthService.context
|
||||
httpGet,
|
||||
vgAuthService.createClickContext()
|
||||
) { response ->
|
||||
if (response.code / 100 != 2) {
|
||||
throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
|
||||
|
||||
@@ -6,7 +6,6 @@ import me.vripper.exception.DownloadException
|
||||
import me.vripper.exception.PostParseException
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.services.RetryPolicyService
|
||||
import me.vripper.services.SettingsService
|
||||
import me.vripper.services.VGAuthService
|
||||
import me.vripper.tasks.Tasks
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
@@ -24,13 +23,12 @@ internal class ThreadLookupAPIParser(private val threadId: Long) : KoinComponent
|
||||
private val cm: HTTPService by inject()
|
||||
private val retryPolicyService: RetryPolicyService by inject()
|
||||
private val vgAuthService: VGAuthService by inject()
|
||||
private val settingsService: SettingsService by inject()
|
||||
|
||||
@Throws(PostParseException::class)
|
||||
fun parse(): ThreadItem {
|
||||
log.debug("Parsing thread $threadId")
|
||||
val httpGet =
|
||||
HttpGet(URIBuilder(settingsService.settings.viperSettings.host + "/vr.php").also {
|
||||
HttpGet(URIBuilder("https://viper.click/vr.php").also {
|
||||
it.setParameter(
|
||||
"t",
|
||||
threadId.toString()
|
||||
@@ -46,9 +44,9 @@ internal class ThreadLookupAPIParser(private val threadId: Long) : KoinComponent
|
||||
)
|
||||
}.get(CheckedSupplier {
|
||||
RequestLimit.getPermit(1)
|
||||
log.debug("Requesting {}", httpGet.uri)
|
||||
log.info("Requesting {}", httpGet.uri)
|
||||
cm.client.execute(
|
||||
httpGet, vgAuthService.context
|
||||
httpGet, vgAuthService.createClickContext()
|
||||
) { response ->
|
||||
if (response.code / 100 != 2) {
|
||||
throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
|
||||
|
||||
@@ -24,31 +24,55 @@ class AboutFragment : Fragment("About") {
|
||||
spacing = 15.0
|
||||
imageview("icons/64x64.png")
|
||||
vbox(spacing = 5.0) {
|
||||
text("VRipper") {
|
||||
text("VRipper ${ApplicationProperties.VERSION}") {
|
||||
style {
|
||||
fontWeight = FontWeight.BOLD
|
||||
fontSize = Dimension(18.0, Dimension.LinearUnits.px)
|
||||
}
|
||||
}
|
||||
text("Version ${ApplicationProperties.VERSION}")
|
||||
text("Developed by dev-claw and VRipper working group")
|
||||
hbox(spacing = 5.0) {
|
||||
hyperlink {
|
||||
imageview("icons/github-mark.png").apply {
|
||||
isPreserveRatio = true
|
||||
fitHeight = 32.0
|
||||
if (widgetsController.currentSettings.darkMode) {
|
||||
imageview("icons/github-mark-white.png").apply {
|
||||
isPreserveRatio = true
|
||||
fitHeight = 32.0
|
||||
}
|
||||
} else {
|
||||
imageview("icons/github-mark.png").apply {
|
||||
isPreserveRatio = true
|
||||
fitHeight = 32.0
|
||||
}
|
||||
}
|
||||
action {
|
||||
openLink("https://github.com/dev-claw/vripper-project")
|
||||
}
|
||||
}
|
||||
hyperlink {
|
||||
imageview("icons/buymeacoffee-logo.png").apply {
|
||||
isPreserveRatio = true
|
||||
fitHeight = 32.0
|
||||
}
|
||||
form {
|
||||
fieldset("Donation") {
|
||||
minWidth = 400.0
|
||||
field {
|
||||
hyperlink {
|
||||
imageview("icons/buymeacoffee-logo.png").apply {
|
||||
isPreserveRatio = true
|
||||
fitHeight = 32.0
|
||||
}
|
||||
action {
|
||||
openLink("https://buymeacoffee.com/devclaw")
|
||||
}
|
||||
}
|
||||
}
|
||||
action {
|
||||
openLink("https://buymeacoffee.com/devclaw")
|
||||
field("ETH:") {
|
||||
textfield("0xDdac82B16dC5E3D742fc915ffF583D8548A301cA") {
|
||||
|
||||
isEditable = false
|
||||
}
|
||||
}
|
||||
field("BTC:") {
|
||||
textfield("bc1qcqudnkrndwyadsjwrxww42svkf8trnzx3c8vlr") {
|
||||
isEditable = false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -38,7 +38,7 @@ class ViperSettingsFragment : Fragment("Viper Settings") {
|
||||
combobox(viperSettingsModel.hostProperty, proxies)
|
||||
}
|
||||
field("Rate limit (requests/s)") {
|
||||
add(Spinner<Int>(1, 6, viperGirlsSettings.requestLimit.toInt()).apply {
|
||||
add(Spinner<Int>(1, 5, viperGirlsSettings.requestLimit.toInt()).apply {
|
||||
viperSettingsModel.requestLimitProperty.bind(valueProperty())
|
||||
isEditable = true
|
||||
atlantafx.base.util.IntegerStringConverter.createFor(this)
|
||||
|
||||
@@ -176,6 +176,8 @@ class GrpcEndpointService : IAppEndpointService {
|
||||
password = settings.viperSettings.password
|
||||
thanks = settings.viperSettings.thanks
|
||||
host = settings.viperSettings.host
|
||||
requestLimit = settings.viperSettings.requestLimit
|
||||
fetchMetadata = settings.viperSettings.fetchMetadata
|
||||
build()
|
||||
}
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 4.7 KiB |
@@ -1,3 +1,3 @@
|
||||
spring.liquibase.enabled=false
|
||||
grpc.enabled=false
|
||||
grpc.enabled=true
|
||||
grpc.port=30000
|
||||
Reference in New Issue
Block a user