Compare commits

...
Author SHA1 Message Date
dev-claw 9f01e7d56c fixes #216 2025-03-04 17:59:36 +01:00
45 changed files with 304 additions and 221 deletions
+7 -2
View File
@@ -108,6 +108,11 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-kotlin</artifactId>
</dependency>
<dependency>
<groupId>com.bucket4j</groupId>
<artifactId>bucket4j_jdk17-core</artifactId>
<version>8.14.0</version>
</dependency>
</dependencies>
<build>
@@ -136,14 +141,14 @@
<executions>
<execution>
<id>compile</id>
<phase>compile</phase>
<phase>process-sources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
<configuration>
<jvmTarget>1.8</jvmTarget>
<jvmTarget>21</jvmTarget>
<compilerPlugins>
<plugin>kotlinx-serialization</plugin>
</compilerPlugins>
@@ -69,7 +69,7 @@ val coreModule = module {
} bind IAppEndpointService::class
single<MetadataService> {
MetadataService(get(), get(), get(), get())
MetadataService(get(), get())
}
single {
AcidimgHost(get(), get(), get())
@@ -18,7 +18,7 @@ import me.vripper.services.RetryPolicyService
import me.vripper.services.SettingsService
import me.vripper.services.VGAuthService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.executorService
import me.vripper.utilities.downloadRunner
import org.jetbrains.exposed.sql.transactions.transaction
import java.util.concurrent.locks.ReentrantLock
import kotlin.concurrent.withLock
@@ -120,10 +120,6 @@ internal class DownloadService(
)
}
toProcess.keys.forEach {
vgAuthService.leaveThanks(it)
}
toProcess.entries.forEach { (post, images) ->
images.forEach { image ->
@@ -231,7 +227,7 @@ internal class DownloadService(
log.debug("Scheduling a job for ${imageDownloadRunnable.imageEntity.url}")
eventBus.publishEvent(QueueStateEvent(QueueState(runningCount(), pendingCount())))
Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicy("Failed to download ${imageDownloadRunnable.imageEntity.url}: "))
.with(executorService)
.with(downloadRunner)
.onFailure {
log.error(
"Failed to download ${imageDownloadRunnable.imageEntity.url} after ${it.attemptCount} tries",
@@ -10,6 +10,7 @@ import me.vripper.host.Host
import me.vripper.host.ImageMimeType
import me.vripper.model.Settings
import me.vripper.services.DataTransaction
import me.vripper.services.VGAuthService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.PathUtils.getExtension
import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
@@ -21,7 +22,6 @@ import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.StandardCopyOption
import java.util.*
import kotlin.Throws
import kotlin.io.path.Path
import kotlin.io.path.pathString
@@ -30,6 +30,7 @@ internal class ImageDownloadRunnable(
) : KoinComponent, CheckedRunnable {
private val log by LoggerDelegate()
private val dataTransaction: DataTransaction by inject()
private val vgauthService: VGAuthService by inject()
private val hosts: List<Host> = getKoin().getAll()
var completed = false
var stopped = false
@@ -46,6 +47,7 @@ internal class ImageDownloadRunnable(
if (post.status != Status.DOWNLOADING) {
post.status = Status.DOWNLOADING
dataTransaction.updatePost(post)
vgauthService.leaveThanks(post)
}
}
log.debug("Getting image url and name from ${imageEntity.url} using ${imageEntity.host}")
@@ -3,7 +3,9 @@ package me.vripper.host
import me.vripper.download.ImageDownloadContext
import me.vripper.exception.HostException
import me.vripper.exception.XpathException
import me.vripper.services.*
import me.vripper.services.DataTransaction
import me.vripper.services.DownloadSpeedService
import me.vripper.services.HTTPService
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.XpathUtils
@@ -46,7 +48,7 @@ internal class AcidimgHost(
)
)
}.also { context.requests.add(it) }
log.debug(String.format("Requesting %s", httpPost))
log.debug(String.format("Requesting %s", httpPost.uri))
val doc = try {
httpService.client.execute(
httpPost, context.httpContext
@@ -20,6 +20,8 @@ data class ViperSettings(
val password: String = "",
val thanks: Boolean = false,
val host: String = "https://vipergirls.to",
val requestLimit: Long = 4,
val fetchMetadata: Boolean = false,
)
@Serializable
@@ -14,7 +14,7 @@ import me.vripper.utilities.ApplicationProperties
import me.vripper.utilities.ApplicationProperties.VRIPPER_DIR
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.PathUtils
import me.vripper.utilities.executorService
import me.vripper.utilities.taskRunner
import org.h2.jdbc.JdbcSQLNonTransientConnectionException
import java.sql.DriverManager
import java.time.Duration
@@ -42,10 +42,10 @@ internal class AppEndpointService(
if (postLinks.isBlank()) {
return
}
val urlList = postLinks.split(Pattern.compile("\\r?\\n")).dropLastWhile { it.isEmpty() }.map { it.trim() }
val urlList = postLinks.split(Pattern.compile("\\r?\\n")).dropLastWhile { it.isBlank() }.map { it.trim() }
.filter { it.isNotEmpty() }
for (link in urlList) {
log.debug("Starting to process thread: $link")
log.debug("Scanning: $link")
if (!link.startsWith(settingsService.settings.viperSettings.host)) {
continue
}
@@ -58,13 +58,13 @@ internal class AppEndpointService(
threadId = m.group(1).toLong()
postId = m.group(4)?.toLong()
if (postId == null) {
executorService.submit(
taskRunner.submit(
ThreadLookupTask(
threadId, settingsService.settings
)
)
} else {
executorService.submit(
taskRunner.submit(
AddPostTask(
listOf(ThreadPostId(threadId, postId))
)
@@ -86,7 +86,7 @@ internal class AppEndpointService(
}
override suspend fun download(posts: List<ThreadPostId>) {
executorService.submit(AddPostTask(posts))
taskRunner.submit(AddPostTask(posts))
}
override suspend fun stopAll(postIdList: List<Long>) {
@@ -182,7 +182,7 @@ internal class AppEndpointService(
}
override suspend fun rename(postId: Long, newName: String) {
executorService.submit {
taskRunner.submit {
synchronized(postId.toString().intern()) {
if (dataTransaction.exists(postId)) {
dataTransaction.findPostByPostId(postId).let { post ->
@@ -1,111 +1,29 @@
package me.vripper.services
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.withPermit
import me.vripper.entities.MetadataEntity
import me.vripper.exception.DownloadException
import me.vripper.exception.VripperException
import me.vripper.tasks.Tasks
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.RequestLimit
import me.vripper.utilities.XpathUtils
import org.apache.hc.client5.http.classic.methods.HttpGet
import org.apache.hc.core5.http.io.entity.EntityUtils
import org.apache.hc.core5.net.URIBuilder
import org.w3c.dom.Node
import java.util.concurrent.atomic.AtomicBoolean
import java.util.stream.Collectors
import me.vripper.tasks.FetchMetadataTask
import me.vripper.utilities.taskRunner
internal class MetadataService(
private val httpService: HTTPService,
private val settingsService: SettingsService,
private val dataTransaction: DataTransaction,
private val vgAuthService: VGAuthService,
private val settingsService: SettingsService,
) {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val dictionary: List<String> = mutableListOf("download", "link", "rapidgator", "filefactory", "filefox")
fun init() {
if (!settingsService.settings.viperSettings.fetchMetadata) {
return
}
dataTransaction.findAllPosts().filter { dataTransaction.findMetadataByPostId(it.postId).isEmpty }
.map { it.postId }.forEach(::fetchMetadata)
}
fun fetchMetadata(postId: Long) {
Tasks.increment()
coroutineScope.launch {
try {
val httpGet = HttpGet(URIBuilder(settingsService.settings.viperSettings.host + "/threads/").also {
it.setParameter(
"p", postId.toString()
)
}.build())
val response = RequestLimit.semaphore.withPermit {
httpService.client.execute(httpGet, vgAuthService.context) {
if (it.code / 100 != 2) {
throw DownloadException("Unexpected response code '${it.code}' for $httpGet")
}
EntityUtils.toString(it.entity)
}
}
val document = HtmlUtils.clean(response)
val postNode: Node = XpathUtils.getAsNode(
document,
"//li[@id='post_$postId']/div[contains(@class, 'postdetails')]",
) ?: throw VripperException("Unable to find post #'$postId'")
val postedBy: String = XpathUtils.getAsNode(
postNode, "./div[contains(@class, 'userinfo')]//a[contains(@class, 'username')]//font"
)?.textContent?.trim()
?: throw VripperException("Unable to find the poster for post #'$postId'")
val node: Node = XpathUtils.getAsNode(
document, java.lang.String.format("//div[@id='post_message_%s']", postId)
) ?: throw VripperException("Unable to locate post content")
val titles = findTitleInContent(node)
val metadataEntity = MetadataEntity(postId, MetadataEntity.Data(postedBy, titles))
dataTransaction.saveMetadata(metadataEntity)
} finally {
Tasks.decrement()
}
}
}
private fun findTitleInContent(node: Node): List<String> {
val altTitle: MutableList<String> = mutableListOf()
findTitle(node, altTitle, AtomicBoolean(true))
return altTitle.stream().distinct().collect(Collectors.toList())
}
private fun findTitle(node: Node, altTitle: MutableList<String>, keepGoing: AtomicBoolean) {
if (!keepGoing.get()) {
if (!settingsService.settings.viperSettings.fetchMetadata) {
return
}
if (node.nodeName == "a" || node.nodeName == "img") {
keepGoing.set(false)
return
}
if (node.nodeType == Node.ELEMENT_NODE) {
for (i in 0 until node.childNodes.length) {
val item = node.childNodes.item(i)
findTitle(item, altTitle, keepGoing)
if (!keepGoing.get()) {
return
}
}
} else if (node.nodeType == Node.TEXT_NODE) {
val text = node.textContent.trim()
if (text.isNotBlank() && dictionary.stream().noneMatch { e ->
text.lowercase().contains(e.lowercase())
}) {
altTitle.add(text)
}
}
taskRunner.submit(
FetchMetadataTask(
postId
)
)
}
}
@@ -188,5 +188,11 @@ class SettingsService(private val eventBus: EventBus) {
"Invalid clipboard monitoring polling rate settings, values must be >= 500"
)
}
if (settings.viperSettings.requestLimit < 1 || settings.viperSettings.requestLimit > 6) {
throw ValidationException(
"Invalid request rate limit, values must be in [1,6]"
)
}
}
}
@@ -13,7 +13,7 @@ import me.vripper.exception.VripperException
import me.vripper.model.Settings
import me.vripper.tasks.LeaveThanksTask
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.executorService
import me.vripper.utilities.taskRunner
import org.apache.hc.client5.http.classic.methods.HttpPost
import org.apache.hc.client5.http.cookie.BasicCookieStore
import org.apache.hc.client5.http.cookie.Cookie
@@ -71,10 +71,6 @@ internal class VGAuthService(
BasicNameValuePair("vb_login_md5password", password)
)
)
it.addHeader("Referer", settings.viperSettings.host)
it.addHeader(
"Host", settings.viperSettings.host.replace("https://", "").replace("http://", "")
)
}
try {
@@ -108,7 +104,7 @@ internal class VGAuthService(
}
fun leaveThanks(postEntity: PostEntity) {
executorService.submit(
taskRunner.submit(
LeaveThanksTask(postEntity, authenticated, context)
)
}
@@ -1,6 +1,5 @@
package me.vripper.tasks
import kotlinx.coroutines.runBlocking
import me.vripper.download.DownloadService
import me.vripper.model.ThreadPostId
import me.vripper.services.DataTransaction
@@ -31,14 +30,14 @@ internal class AddPostTask(private val items: List<ThreadPostId>) : KoinComponen
}
val link =
"https://${settingsService.settings.viperSettings.host}/threads/$threadId?p=$postId&viewfull=1#post$postId"
"${settingsService.settings.viperSettings.host}/threads/$threadId?p=$postId&viewfull=1#post$postId"
val cachedThread = threadCacheService.getIfPresent(threadId)
val threadItem = cachedThread ?: runBlocking {
val threadItem = cachedThread ?:
PostLookupAPIParser(
threadId, postId
).parse()
}
if (threadItem == null) {
log.error("Failed to load $link")
@@ -0,0 +1,110 @@
package me.vripper.tasks
import me.vripper.entities.MetadataEntity
import me.vripper.exception.DownloadException
import me.vripper.exception.VripperException
import me.vripper.services.DataTransaction
import me.vripper.services.HTTPService
import me.vripper.services.SettingsService
import me.vripper.services.VGAuthService
import me.vripper.utilities.HtmlUtils
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.RequestLimit
import me.vripper.utilities.XpathUtils
import org.apache.hc.client5.http.classic.methods.HttpGet
import org.apache.hc.core5.http.io.entity.EntityUtils
import org.apache.hc.core5.net.URIBuilder
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import org.w3c.dom.Node
import java.util.concurrent.atomic.AtomicBoolean
import java.util.stream.Collectors
internal class FetchMetadataTask(
private val postId: Long,
) : KoinComponent, Runnable {
private val dictionary: List<String> = mutableListOf("download", "link", "rapidgator", "filefactory", "filefox")
private val log by LoggerDelegate()
private val settingsService: SettingsService by inject()
private val vgAuthService: VGAuthService by inject()
private val httpService: HTTPService by inject()
private val dataTransaction: DataTransaction by inject()
override fun run() {
try {
if (!settingsService.settings.viperSettings.fetchMetadata) {
log.debug("Fetching metadata is disabled")
return
}
Tasks.increment()
val httpGet = HttpGet(URIBuilder(settingsService.settings.viperSettings.host + "/threads/").also {
it.setParameter(
"p", postId.toString()
)
}.build())
RequestLimit.getPermit(1)
log.debug("Requesting {}", httpGet.uri)
val response = httpService.client.execute(httpGet, vgAuthService.context) {
if (it.code / 100 != 2) {
throw DownloadException("Unexpected response code '${it.code}' for $httpGet")
}
EntityUtils.toString(it.entity)
}
val document = HtmlUtils.clean(response)
val postNode: Node = XpathUtils.getAsNode(
document,
"//li[@id='post_$postId']/div[contains(@class, 'postdetails')]",
) ?: throw VripperException("Unable to find post #'$postId'")
val postedBy: String = XpathUtils.getAsNode(
postNode, "./div[contains(@class, 'userinfo')]//a[contains(@class, 'username')]//font"
)?.textContent?.trim()
?: throw VripperException("Unable to find the poster for post #'$postId'")
val node: Node = XpathUtils.getAsNode(
document, java.lang.String.format("//div[@id='post_message_%s']", postId)
) ?: throw VripperException("Unable to locate post content")
val titles = findTitleInContent(node)
val metadataEntity = MetadataEntity(postId, MetadataEntity.Data(postedBy, titles))
dataTransaction.saveMetadata(metadataEntity)
} finally {
Tasks.decrement()
}
}
private fun findTitleInContent(node: Node): List<String> {
val altTitle: MutableList<String> = mutableListOf()
findTitle(node, altTitle, AtomicBoolean(true))
return altTitle.stream().distinct().collect(Collectors.toList())
}
private fun findTitle(node: Node, altTitle: MutableList<String>, keepGoing: AtomicBoolean) {
if (!keepGoing.get()) {
return
}
if (node.nodeName == "a" || node.nodeName == "img") {
keepGoing.set(false)
return
}
if (node.nodeType == Node.ELEMENT_NODE) {
for (i in 0 until node.childNodes.length) {
val item = node.childNodes.item(i)
findTitle(item, altTitle, keepGoing)
if (!keepGoing.get()) {
return
}
}
} else if (node.nodeType == Node.TEXT_NODE) {
val text = node.textContent.trim()
if (text.isNotBlank() && dictionary.stream().noneMatch { e ->
text.lowercase().contains(e.lowercase())
}) {
altTitle.add(text)
}
}
}
}
@@ -1,10 +1,11 @@
package me.vripper.tasks
import me.vripper.entities.PostEntity
import me.vripper.services.DataTransaction
import me.vripper.exception.VripperException
import me.vripper.services.HTTPService
import me.vripper.services.SettingsService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.RequestLimit
import org.apache.hc.client5.http.classic.methods.HttpPost
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
import org.apache.hc.client5.http.protocol.HttpClientContext
@@ -20,7 +21,6 @@ internal class LeaveThanksTask(
private val log by LoggerDelegate()
private val cm: HTTPService by inject()
private val settingsService: SettingsService by inject()
private val dataTransaction: DataTransaction by inject()
override fun run() {
try {
@@ -31,25 +31,23 @@ internal class LeaveThanksTask(
if (!settingsService.settings.viperSettings.thanks) {
return
}
val postThanks =
HttpPost("${settingsService.settings.viperSettings.host}/post_thanks.php").also {
it.entity = UrlEncodedFormEntity(
listOf(
BasicNameValuePair("do", "post_thanks_add"),
BasicNameValuePair("using_ajax", "1"),
BasicNameValuePair("p", postEntity.postId.toString()),
BasicNameValuePair("securitytoken", postEntity.token)
)
)
it.addHeader("Referer", settingsService.settings.viperSettings.host)
it.addHeader(
"Host",
settingsService.settings.viperSettings.host.replace("https://", "")
.replace("http://", "")
val postThanks = HttpPost("${settingsService.settings.viperSettings.host}/post_thanks.php").also {
it.entity = UrlEncodedFormEntity(
listOf(
BasicNameValuePair("do", "post_thanks_add"),
BasicNameValuePair("using_ajax", "1"),
BasicNameValuePair("p", postEntity.postId.toString()),
BasicNameValuePair("securitytoken", postEntity.token)
)
)
}
RequestLimit.getPermit(1)
log.debug("Requesting {}", postThanks.uri)
cm.client.execute(postThanks, context) { response ->
if (response.code / 100 != 2) {
throw VripperException("Unexpected response code '${response.code}' for $postThanks")
}
cm.client.execute(postThanks, context) { }
}
} catch (e: Exception) {
log.error("Failed to leave a thanks for $postEntity", e)
} finally {
@@ -7,7 +7,7 @@ import me.vripper.services.DataTransaction
import me.vripper.services.SettingsService
import me.vripper.services.ThreadCacheService
import me.vripper.utilities.LoggerDelegate
import me.vripper.utilities.executorService
import me.vripper.utilities.taskRunner
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
@@ -33,7 +33,7 @@ internal class ThreadLookupTask(private val threadId: Long, private val settings
}
if (threadLookupResult.postItemList.size <= settings.downloadSettings.autoQueueThreshold) {
executorService.submit(
taskRunner.submit(
AddPostTask(threadLookupResult.postItemList.map {
ThreadPostId(
it.threadId, it.postId
@@ -1,7 +1,42 @@
package me.vripper.utilities
import kotlinx.coroutines.sync.Semaphore
import io.github.bucket4j.Bucket
import io.github.bucket4j.BucketConfiguration
import io.github.bucket4j.TokensInheritanceStrategy
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.flow.collectLatest
import kotlinx.coroutines.flow.filterIsInstance
import kotlinx.coroutines.launch
import me.vripper.event.EventBus
import me.vripper.event.SettingsUpdateEvent
import me.vripper.services.SettingsService
import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.time.Duration
internal object RequestLimit {
val semaphore: Semaphore = Semaphore(2)
internal object RequestLimit : KoinComponent {
private val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val settingsService: SettingsService by inject()
private val eventBus: EventBus by inject()
private val bucket: Bucket = Bucket.builder().addLimit {
it.capacity(settingsService.settings.viperSettings.requestLimit)
.refillGreedy(settingsService.settings.viperSettings.requestLimit, Duration.ofSeconds(1))
}.build()
init {
coroutineScope.launch {
eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collectLatest { event ->
bucket.replaceConfiguration(BucketConfiguration.builder().addLimit {
it.capacity(event.settings.viperSettings.requestLimit)
.refillGreedy(event.settings.viperSettings.requestLimit, Duration.ofSeconds(1))
}.build(), TokensInheritanceStrategy.RESET)
}
}
}
fun getPermit(tokens: Long) {
bucket.asBlocking().consume(tokens)
}
}
@@ -3,4 +3,5 @@ package me.vripper.utilities
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
val executorService: ExecutorService = Executors.newVirtualThreadPerTaskExecutor()
val taskRunner: ExecutorService = Executors.newFixedThreadPool(6)
val downloadRunner: ExecutorService = Executors.newVirtualThreadPerTaskExecutor()
@@ -2,8 +2,6 @@ package me.vripper.vgapi
import dev.failsafe.Failsafe
import dev.failsafe.function.CheckedSupplier
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.withPermit
import me.vripper.exception.DownloadException
import me.vripper.exception.PostParseException
import me.vripper.services.HTTPService
@@ -20,7 +18,6 @@ import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.io.ByteArrayInputStream
import javax.xml.parsers.SAXParserFactory
import kotlin.Throws
internal class PostLookupAPIParser(private val threadId: Long, private val postId: Long) : KoinComponent {
private val log by LoggerDelegate()
@@ -39,7 +36,6 @@ internal class PostLookupAPIParser(private val threadId: Long, private val postI
)
}.build())
val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler()
log.debug("Requesting {}", httpGet)
Tasks.increment()
return try {
Failsafe.with(retryPolicyService.buildRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
@@ -47,22 +43,21 @@ internal class PostLookupAPIParser(private val threadId: Long, private val postI
"Failed to process thread $threadId, post $postId", it.exception
)
}.get(CheckedSupplier {
runBlocking {
RequestLimit.semaphore.withPermit {
httpService.client.execute(
httpGet, vgAuthService.context
) { response ->
if (response.code / 100 != 2) {
throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
}
ByteArrayInputStream(EntityUtils.toByteArray(response.entity))
}
}.use {
RequestLimit.getPermit(1)
log.debug("Requesting {}", httpGet.uri)
httpService.client.execute(
httpGet, vgAuthService.context
) { response ->
if (response.code / 100 != 2) {
throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
}
ByteArrayInputStream(EntityUtils.toByteArray(response.entity)).use {
factory.newSAXParser()
.parse(it, threadLookupAPIResponseHandler)
threadLookupAPIResponseHandler.result
}
}
})
} catch (e: Exception) {
throw PostParseException(e)
@@ -2,8 +2,6 @@ package me.vripper.vgapi
import dev.failsafe.Failsafe
import dev.failsafe.function.CheckedSupplier
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.withPermit
import me.vripper.exception.DownloadException
import me.vripper.exception.PostParseException
import me.vripper.services.HTTPService
@@ -20,7 +18,6 @@ import org.koin.core.component.KoinComponent
import org.koin.core.component.inject
import java.io.ByteArrayInputStream
import javax.xml.parsers.SAXParserFactory
import kotlin.Throws
internal class ThreadLookupAPIParser(private val threadId: Long) : KoinComponent {
private val log by LoggerDelegate()
@@ -40,7 +37,6 @@ internal class ThreadLookupAPIParser(private val threadId: Long) : KoinComponent
)
}.build())
val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler()
log.debug("Requesting {}", httpGet)
Tasks.increment()
return try {
Failsafe.with(retryPolicyService.buildRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
@@ -49,17 +45,15 @@ internal class ThreadLookupAPIParser(private val threadId: Long) : KoinComponent
it.exception
)
}.get(CheckedSupplier {
runBlocking {
RequestLimit.semaphore.withPermit {
cm.client.execute(
httpGet, vgAuthService.context
) { response ->
if (response.code / 100 != 2) {
throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
}
ByteArrayInputStream(EntityUtils.toByteArray(response.entity))
}
}.use {
RequestLimit.getPermit(1)
log.debug("Requesting {}", httpGet.uri)
cm.client.execute(
httpGet, vgAuthService.context
) { response ->
if (response.code / 100 != 2) {
throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
}
ByteArrayInputStream(EntityUtils.toByteArray(response.entity)).use {
factory.newSAXParser().parse(
it,
threadLookupAPIResponseHandler
@@ -7,6 +7,8 @@ message ViperSettings {
string password = 3;
bool thanks = 4;
string host = 5;
int64 requestLimit = 6;
bool fetchMetadata = 7;
}
message DownloadSettings {
+1 -2
View File
@@ -19,8 +19,7 @@
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${VRIPPER_DIR}/vripper.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
<maxFileSize>10MB</maxFileSize>
<maxHistory>7</maxHistory>
<totalSizeCap>1GB</totalSizeCap>
<maxHistory>5</maxHistory>
</rollingPolicy>
</appender>
+2 -2
View File
@@ -160,14 +160,14 @@
<executions>
<execution>
<id>compile</id>
<phase>compile</phase>
<phase>process-sources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
<configuration>
<jvmTarget>1.8</jvmTarget>
<jvmTarget>21</jvmTarget>
<compilerPlugins>
<plugin>kotlinx-serialization</plugin>
</compilerPlugins>
@@ -31,7 +31,7 @@ class VripperGuiApplication : App(
private val log by LoggerDelegate()
private var initialized = false
private val widgetsController: WidgetsController by inject()
private val coroutineScope = CoroutineScope(SupervisorJob())
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
init {
APP_INSTANCE = this
@@ -6,16 +6,13 @@ import javafx.geometry.Pos
import javafx.scene.control.TextArea
import javafx.scene.layout.Priority
import javafx.scene.layout.VBox
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancel
import kotlinx.coroutines.launch
import kotlinx.coroutines.*
import me.vripper.gui.controller.PostController
import tornadofx.*
class AddLinksFragment : Fragment("Add thread links") {
private val coroutineScope = CoroutineScope(SupervisorJob())
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val textAreaProperty = SimpleStringProperty()
private val postController: PostController by inject()
lateinit var input: TextArea
@@ -9,7 +9,7 @@ import tornadofx.*
class ConnectionSettingsFragment : Fragment("Connection Settings") {
private val settingsController: SettingsController by inject()
private val coroutineScope = CoroutineScope(SupervisorJob())
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private lateinit var connectionSettings: ConnectionSettings
val connectionSettingsModel = ConnectionSettingsModel()
@@ -13,7 +13,7 @@ class DownloadSettingsFragment : Fragment("Download Settings") {
private val settingsController: SettingsController by inject()
private val widgetsController: WidgetsController by inject()
private val coroutineScope = CoroutineScope(SupervisorJob())
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private lateinit var downloadSettings: DownloadSettings
val downloadSettingsModel = DownloadSettingsModel()
@@ -6,10 +6,7 @@ import javafx.scene.control.Alert
import javafx.scene.control.TabPane
import javafx.scene.layout.Priority
import javafx.scene.layout.VBox
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancel
import kotlinx.coroutines.launch
import kotlinx.coroutines.*
import me.vripper.exception.ValidationException
import me.vripper.gui.controller.SettingsController
import org.kordamp.ikonli.feather.Feather
@@ -20,7 +17,7 @@ import tornadofx.*
class SettingsFragment : Fragment("Settings") {
private val settingsController: SettingsController by inject()
private val coroutineScope = CoroutineScope(SupervisorJob())
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val downloadSettingsFragment: DownloadSettingsFragment = find()
private val connectionSettingsFragment: ConnectionSettingsFragment = find()
private val viperSettingsFragment: ViperSettingsFragment = find()
@@ -13,7 +13,7 @@ import tornadofx.*
class SystemSettingsFragment : Fragment("System Settings") {
private val settingsController: SettingsController by inject()
private val widgetsController: WidgetsController by inject()
private val coroutineScope = CoroutineScope(SupervisorJob())
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private lateinit var systemSettings: SystemSettings
val systemSettingsModel = SystemSettingsModel()
@@ -2,6 +2,7 @@ package me.vripper.gui.components.fragments
import atlantafx.base.controls.ToggleSwitch
import javafx.collections.FXCollections
import javafx.scene.control.Spinner
import kotlinx.coroutines.*
import me.vripper.gui.controller.SettingsController
import me.vripper.gui.model.settings.ViperSettingsModel
@@ -11,7 +12,7 @@ import tornadofx.*
class ViperSettingsFragment : Fragment("Viper Settings") {
private val settingsController: SettingsController by inject()
private val proxies = FXCollections.observableArrayList<String>()
private val coroutineScope = CoroutineScope(SupervisorJob())
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private lateinit var viperGirlsSettings: ViperSettings
val viperSettingsModel = ViperSettingsModel()
@@ -25,16 +26,30 @@ class ViperSettingsFragment : Fragment("Viper Settings") {
viperSettingsModel.password = viperGirlsSettings.password
viperSettingsModel.thanks = viperGirlsSettings.thanks
viperSettingsModel.host = viperGirlsSettings.host
viperSettingsModel.requestLimit = viperGirlsSettings.requestLimit
viperSettingsModel.fetchMetadata = viperGirlsSettings.fetchMetadata
proxies.addAll(settingsController.getProxies())
}.await()
runLater {
with(root) {
form {
fieldset {
field("Select a proxy") {
field("Viper domain") {
combobox(viperSettingsModel.hostProperty, proxies)
}
field("Enable ViperGirls Authentication") {
field("Rate limit (requests/s)") {
add(Spinner<Int>(1, 6, viperGirlsSettings.requestLimit.toInt()).apply {
viperSettingsModel.requestLimitProperty.bind(valueProperty())
isEditable = true
atlantafx.base.util.IntegerStringConverter.createFor(this)
})
}
field("Fetch Metadata") {
checkbox {
bind(viperSettingsModel.fetchMetadataProperty)
}
}
field("Authentication") {
add(ToggleSwitch().apply {
isSelected = viperGirlsSettings.login
viperSettingsModel.loginProperty.bind(selectedProperty())
@@ -42,13 +57,13 @@ class ViperSettingsFragment : Fragment("Viper Settings") {
}
fieldset {
visibleWhen(viperSettingsModel.loginProperty)
field("ViperGirls Username") {
field("Username") {
textfield(viperSettingsModel.usernameProperty)
}
field("ViperGirls Password") {
field("Password") {
passwordfield(viperSettingsModel.passwordProperty)
}
field("Leave like") {
field("Leave likes") {
checkbox {
bind(viperSettingsModel.thanksProperty)
}
@@ -25,7 +25,7 @@ class ActionBarView : View() {
private val downloadActiveProperty = SimpleBooleanProperty(true)
private val postController: PostController by inject()
private val postsTableView: PostsTableView by inject()
private val coroutineScope = CoroutineScope(SupervisorJob())
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val grpcEndpointService: IAppEndpointService by di("remoteAppEndpointService")
private val localEndpointService: IAppEndpointService by di("localAppEndpointService")
private val running = SimpleIntegerProperty(0)
@@ -37,7 +37,7 @@ class ImagesTableView : View("Photos") {
private val widgetsController: WidgetsController by inject()
private val localAppEndpointService: IAppEndpointService by di("localAppEndpointService")
private val remoteAppEndpointService: IAppEndpointService by di("remoteAppEndpointService")
private val coroutineScope = CoroutineScope(SupervisorJob())
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val items: ObservableList<ImageModel> = FXCollections.observableArrayList()
private var preview: Preview? = null
@@ -18,7 +18,7 @@ class LoadingView : View("VRipper") {
private val widgetsController: WidgetsController by inject()
private val grpcEndpointService: GrpcEndpointService by di("remoteAppEndpointService")
private val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
override val root = borderpane {}
@@ -26,7 +26,7 @@ class LogTableView : View() {
private val localAppEndpointService: IAppEndpointService by di("localAppEndpointService")
private val remoteAppEndpointService: IAppEndpointService by di("remoteAppEndpointService")
private val tableView: TableView<LogModel>
private val coroutineScope = CoroutineScope(SupervisorJob())
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val items: ObservableList<LogModel> = FXCollections.observableArrayList()
private var maxLogEvent = 0
@@ -27,7 +27,7 @@ import tornadofx.*
class MenuBarView : View() {
private val logger by LoggerDelegate()
private val coroutineScope = CoroutineScope(SupervisorJob())
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val downloadActiveProperty = SimpleBooleanProperty(false)
private val postsTableView: PostsTableView by inject()
private val widgetsController: WidgetsController by inject()
@@ -18,7 +18,7 @@ import tornadofx.*
class PostInfoView : View() {
private val logger by LoggerDelegate()
private val postController: PostController by inject()
private val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob())
private val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val imagesTableView: ImagesTableView by inject()
private val postModel: PostModel = PostModel(
-1, "", 0.0, "", "", 0, 0, "", "", -1, "", "", "", emptyList(), emptyList(), "", 0
@@ -37,7 +37,7 @@ import kotlin.io.path.Path
class PostsTableView : View() {
private val coroutineScope = CoroutineScope(SupervisorJob())
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val postController: PostController by inject()
private val widgetsController: WidgetsController by inject()
private val clipboardService: ClipboardService by inject()
@@ -18,7 +18,7 @@ import tornadofx.*
class StatusBarView : View("Status bar") {
private val logger by LoggerDelegate()
private val widgetsController: WidgetsController by inject()
private val coroutineScope = CoroutineScope(SupervisorJob())
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val grpcEndpointService: IAppEndpointService by di("remoteAppEndpointService")
private val localEndpointService: IAppEndpointService by di("localAppEndpointService")
private val remoteText = SimpleStringProperty()
@@ -24,7 +24,7 @@ import tornadofx.*
class ThreadTableView : View() {
private val coroutineScope = CoroutineScope(SupervisorJob())
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val threadController: ThreadController by inject()
private val widgetsController: WidgetsController by inject()
private val mainView: MainView by inject()
@@ -59,14 +59,16 @@ class SettingsController : Controller() {
viperSettingsModel.password,
viperSettingsModel.thanks,
viperSettingsModel.host,
viperSettingsModel.requestLimit,
viperSettingsModel.fetchMetadata,
),
systemSettings =
SystemSettings(
systemSettingsModel.tempPath,
systemSettingsModel.enable,
systemSettingsModel.pollingRate,
systemSettingsModel.logEntries
)
SystemSettings(
systemSettingsModel.tempPath,
systemSettingsModel.enable,
systemSettingsModel.pollingRate,
systemSettingsModel.logEntries
)
)
)
}
@@ -1,6 +1,7 @@
package me.vripper.gui.model.settings
import javafx.beans.property.SimpleBooleanProperty
import javafx.beans.property.SimpleLongProperty
import javafx.beans.property.SimpleStringProperty
import tornadofx.getValue
import tornadofx.setValue
@@ -20,4 +21,10 @@ class ViperSettingsModel {
val hostProperty = SimpleStringProperty()
var host: String by hostProperty
val requestLimitProperty = SimpleLongProperty()
var requestLimit: Long by requestLimitProperty
val fetchMetadataProperty = SimpleBooleanProperty()
var fetchMetadata: Boolean by fetchMetadataProperty
}
@@ -14,7 +14,7 @@ import tornadofx.runLater
class ClipboardService : Controller() {
private val logger by LoggerDelegate()
private var current: String? = null
private var coroutineScope = CoroutineScope(SupervisorJob())
private var coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private var pollJob: Job? = null
private var subscribeJob: Job? = null
@@ -241,6 +241,7 @@ class GrpcEndpointService : IAppEndpointService {
password = settings.viperSettings.password,
thanks = settings.viperSettings.thanks,
host = settings.viperSettings.host,
requestLimit = settings.viperSettings.requestLimit,
),
downloadSettings = DownloadSettings(
downloadPath = settings.downloadSettings.downloadPath,
@@ -25,7 +25,7 @@ class Preview(owner: Stage, private val images: List<String>, private val cacheP
private var hBox: HBox = HBox().apply { spacing = 5.0; alignment = Pos.BOTTOM_CENTER }
private val log by LoggerDelegate()
private val coroutineScope = CoroutineScope(SupervisorJob())
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
val previewPopup = Popup()
private val cache: LoadingCache<String, ByteArray> = Caffeine
+1 -1
View File
@@ -106,7 +106,7 @@
<executions>
<execution>
<id>compile</id>
<phase>compile</phase>
<phase>process-sources</phase>
<goals>
<goal>compile</goal>
</goals>
@@ -196,6 +196,8 @@ class GrpcServerAppEndpointService : EndpointServiceGrpcKt.EndpointServiceCorout
password = request.viperSettings.password,
thanks = request.viperSettings.thanks,
host = request.viperSettings.host,
requestLimit = request.viperSettings.requestLimit,
fetchMetadata = request.viperSettings.fetchMetadata,
), systemSettings = SystemSettings(
tempPath = request.systemSettings.tempPath,
enableClipboardMonitoring = request.systemSettings.enableClipboardMonitoring,
@@ -228,6 +230,8 @@ class GrpcServerAppEndpointService : EndpointServiceGrpcKt.EndpointServiceCorout
password = settings.viperSettings.password
thanks = settings.viperSettings.thanks
host = settings.viperSettings.host
requestLimit = settings.viperSettings.requestLimit
fetchMetadata = settings.viperSettings.fetchMetadata
build()
}
@@ -14,7 +14,7 @@ class DataBroadcast(
private val template: SimpMessagingTemplate
) : KoinComponent {
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
private val appEndpointService: IAppEndpointService by inject(named("localAppEndpointService"))
@OptIn(FlowPreview::class)