mirror of
https://github.com/dev-claw/vripper-project.git
synced 2026-08-19 08:35:41 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 dev-claw
|
ca77074ef6 | ||
|
dev-clawandGitHub
|
3cb251cb36 | ||
|
dev-clawandGitHub
|
73f478c1bc | ||
|
dev-claw
|
bb1dfbfee1 | ||
|
dev-clawandGitHub
|
a89fe8571c | ||
|
dev-clawandGitHub
|
44c8e443f4 | ||
|
dev-claw
|
7ddc5d00b9 | ||
|
dev-claw
|
00039e230b | ||
|
dev-clawandGitHub
|
f4821aafac |
@@ -57,9 +57,9 @@ jobs:
|
||||
name: Package for Linux
|
||||
run: |
|
||||
cd jpackage
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.png
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-linux.cfg" --type deb
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-linux.cfg" --type rpm
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.png
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-linux.cfg" --resource-dir resources --type deb
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-linux.cfg" --resource-dir resources --type rpm
|
||||
mv dist/vripper-${{ github.event.release.tag_name }}-1.x86_64.rpm dist/vripper-linux-${{ github.event.release.tag_name }}.x86_64.rpm
|
||||
mv dist/vripper_${{ github.event.release.tag_name }}-1_amd64.deb dist/vripper-linux-${{ github.event.release.tag_name }}_amd64.deb
|
||||
|
||||
@@ -67,9 +67,9 @@ jobs:
|
||||
name: Package for Windows
|
||||
run: |
|
||||
cd jpackage
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.ico
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-windows.cfg" --type msi
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-windows.cfg" --type exe
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.ico
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-windows.cfg" --resource-dir resources --type msi
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-windows.cfg" --resource-dir resources --type exe
|
||||
cd dist
|
||||
ren VRipper-${{ github.event.release.tag_name }}.msi vripper-windows-installer-${{ github.event.release.tag_name }}.msi
|
||||
ren VRipper-${{ github.event.release.tag_name }}.exe vripper-windows-installer-${{ github.event.release.tag_name }}.exe
|
||||
@@ -78,9 +78,9 @@ jobs:
|
||||
name: Package for macOS(arm64)
|
||||
run: |
|
||||
cd jpackage
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.icns
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type pkg
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type dmg
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.icns
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type pkg
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type dmg
|
||||
mv dist/VRipper-${{ github.event.release.tag_name }}.pkg dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.pkg
|
||||
mv dist/VRipper-${{ github.event.release.tag_name }}.dmg dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.dmg
|
||||
|
||||
@@ -88,9 +88,9 @@ jobs:
|
||||
name: Package for macOS(x86_64)
|
||||
run: |
|
||||
cd jpackage
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.icns
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type pkg
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type dmg
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon resources/VRipper.icns
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type pkg
|
||||
jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --resource-dir resources --type dmg
|
||||
mv dist/VRipper-${{ github.event.release.tag_name }}.pkg dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.pkg
|
||||
mv dist/VRipper-${{ github.event.release.tag_name }}.dmg dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.dmg
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ Direct access to `vipergirls.to` domain.
|
||||
|
||||
<img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/windows_logo.svg" width="100%" height="75"> Windows | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS (Intel) | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS (Apple silicon) | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/linux_logo.svg" width="100%" height="75"> Linux | <img src="https://images.vexels.com/media/users/3/166401/isolated/preview/b82aa7ac3f736dd78570dd3fa3fa9e24-java-programming-language-icon-by-vexels.png" width="100%" height="75"> Java
|
||||
:---:|:---:|:---:|:---:|:---:
|
||||
[Installer (EXE)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-windows-installer-6.5.3.exe) <br /> [Installer (MSI)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-windows-installer-6.5.3.msi) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-windows-portable-6.5.3.zip) | [Installer (DMG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-macos-6.5.3.x86_64.dmg) <br /> [Installer (PKG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-macos-6.5.3.x86_64.pkg) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-macos-portable-6.5.3.x86_64.zip) | [Installer (DMG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-macos-6.5.3.arm64.dmg) <br /> [Installer (PKG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-macos-6.5.3.arm64.pkg) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-macos-portable-6.5.3.arm64.zip) | [Linux (amd64) (DEB)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-linux-6.5.3_amd64.deb) <br /> [Linux (x86_64) (RPM)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-linux-6.5.3.x86_64.rpm) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-linux-portable-6.5.3.zip) | [Java GUI (noarch)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-noarch-gui-6.5.3.jar) <br /> [Java Web (noarch)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-noarch-web-6.5.3.jar)
|
||||
[Installer (EXE)](https://github.com/dev-claw/vripper-project/releases/download/6.5.6/vripper-windows-installer-6.5.6.exe) <br /> [Installer (MSI)](https://github.com/dev-claw/vripper-project/releases/download/6.5.6/vripper-windows-installer-6.5.6.msi) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.6/vripper-windows-portable-6.5.6.zip) | [Installer (DMG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.6/vripper-macos-6.5.6.x86_64.dmg) <br /> [Installer (PKG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.6/vripper-macos-6.5.6.x86_64.pkg) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.6/vripper-macos-portable-6.5.6.x86_64.zip) | [Installer (DMG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.6/vripper-macos-6.5.6.arm64.dmg) <br /> [Installer (PKG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.6/vripper-macos-6.5.6.arm64.pkg) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.6/vripper-macos-portable-6.5.6.arm64.zip) | [Linux (amd64) (DEB)](https://github.com/dev-claw/vripper-project/releases/download/6.5.6/vripper-linux-6.5.3_amd64.deb) <br /> [Linux (x86_64) (RPM)](https://github.com/dev-claw/vripper-project/releases/download/6.5.6/vripper-linux-6.5.6.x86_64.rpm) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.6/vripper-linux-portable-6.5.6.zip) | [Java GUI (noarch)](https://github.com/dev-claw/vripper-project/releases/download/6.5.6/vripper-noarch-gui-6.5.6.jar) <br /> [Java Web (noarch)](https://github.com/dev-claw/vripper-project/releases/download/6.5.6/vripper-noarch-web-6.5.6.jar)
|
||||
|
||||
Source code and previous versions are available on
|
||||
the [Releases page](https://github.com/dev-claw/vripper-project/releases).
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
--icon icon.png
|
||||
--java-options "-Dvripper.portable=false"
|
||||
--linux-package-name vripper
|
||||
--linux-app-release 1
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
--icon icon.icns
|
||||
--java-options "-Dvripper.portable=false"
|
||||
--mac-package-identifier me.mnlr.vripper.vripper-gui
|
||||
--mac-package-name VRipper
|
||||
@@ -1,4 +1,3 @@
|
||||
--icon icon.ico
|
||||
--java-options "-Dvripper.portable=false"
|
||||
--win-dir-chooser
|
||||
--win-menu
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
[Desktop Entry]
|
||||
Name=VRipper
|
||||
Comment=Image ripper tool for vipergirls
|
||||
Exec=/opt/vripper/bin/VRipper
|
||||
Icon=/opt/vripper/lib/VRipper.png
|
||||
Terminal=false
|
||||
Type=Application
|
||||
Categories=Utility
|
||||
MimeType=
|
||||
StartupWMClass=me.vripper.gui.VripperGuiApplication
|
||||
|
Before Width: | Height: | Size: 303 KiB After Width: | Height: | Size: 303 KiB |
|
Before Width: | Height: | Size: 18 KiB After Width: | Height: | Size: 18 KiB |
@@ -15,7 +15,7 @@
|
||||
<maven.compiler.source>${java.version}</maven.compiler.source>
|
||||
<maven.compiler.target>${java.version}</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<revision>6.5.3</revision>
|
||||
<revision>6.5.6</revision>
|
||||
<kotlin.version>2.0.21</kotlin.version>
|
||||
<slf4j.version>2.0.16</slf4j.version>
|
||||
<logback.version>1.5.12</logback.version>
|
||||
|
||||
@@ -71,51 +71,51 @@ val coreModule = module {
|
||||
MetadataService(get(), get())
|
||||
}
|
||||
single {
|
||||
AcidimgHost(get(), get(), get())
|
||||
AcidimgHost(get())
|
||||
} bind Host::class
|
||||
single {
|
||||
DPicMeHost(get(), get(), get())
|
||||
DPicMeHost()
|
||||
} bind Host::class
|
||||
single {
|
||||
ImageBamHost(get(), get(), get())
|
||||
ImageBamHost()
|
||||
} bind Host::class
|
||||
single {
|
||||
ImageTwistHost(get(), get(), get())
|
||||
ImageTwistHost()
|
||||
} bind Host::class
|
||||
single {
|
||||
ImageVenueHost(get(), get(), get())
|
||||
ImageVenueHost()
|
||||
} bind Host::class
|
||||
single {
|
||||
ImageZillaHost(get(), get(), get())
|
||||
ImageZillaHost()
|
||||
} bind Host::class
|
||||
single {
|
||||
ImgboxHost(get(), get(), get())
|
||||
ImgboxHost()
|
||||
} bind Host::class
|
||||
single {
|
||||
ImgSpiceHost(get(), get(), get())
|
||||
ImgSpiceHost()
|
||||
} bind Host::class
|
||||
single {
|
||||
ImxHost(get(), get(), get())
|
||||
ImxHost()
|
||||
} bind Host::class
|
||||
single {
|
||||
PimpandhostHost(get(), get(), get())
|
||||
PimpandhostHost()
|
||||
} bind Host::class
|
||||
single {
|
||||
PixhostHost(get(), get(), get())
|
||||
PixhostHost()
|
||||
} bind Host::class
|
||||
single {
|
||||
PixRouteHost(get(), get(), get())
|
||||
PixRouteHost()
|
||||
} bind Host::class
|
||||
single {
|
||||
PixxxelsHost(get(), get(), get())
|
||||
PixxxelsHost()
|
||||
} bind Host::class
|
||||
single {
|
||||
PostImgHost(get(), get(), get())
|
||||
PostImgHost()
|
||||
} bind Host::class
|
||||
single {
|
||||
TurboImageHost(get(), get(), get())
|
||||
TurboImageHost()
|
||||
} bind Host::class
|
||||
single {
|
||||
ViprImHost(get(), get(), get())
|
||||
ViprImHost()
|
||||
} bind Host::class
|
||||
}
|
||||
@@ -1,10 +1,9 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.HtmlUtils
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
@@ -12,33 +11,31 @@ import me.vripper.utilities.XpathUtils
|
||||
import org.apache.hc.client5.http.classic.methods.HttpPost
|
||||
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
|
||||
import org.apache.hc.core5.http.message.BasicNameValuePair
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
|
||||
internal class AcidimgHost(
|
||||
private val httpService: HTTPService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host("acidimg.cc", 0, httpService, dataTransaction, downloadSpeedService) {
|
||||
) : Host("acidimg.cc", 0) {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String, document: Document, context: ImageDownloadContext
|
||||
image: ImageEntity, context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val document = fetchDocument(image.url, context)
|
||||
try {
|
||||
log.debug(
|
||||
String.format(
|
||||
"Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url
|
||||
"Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, image.url
|
||||
)
|
||||
)
|
||||
XpathUtils.getAsNode(document, CONTINUE_BUTTON_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
}
|
||||
log.debug(String.format("Click button found for %s", url))
|
||||
val httpPost = HttpPost(url).also {
|
||||
it.addHeader("Referer", url)
|
||||
log.debug(String.format("Click button found for %s", image.url))
|
||||
val httpPost = HttpPost(image.url).also {
|
||||
it.addHeader("Referer", image.url)
|
||||
it.entity = UrlEncodedFormEntity(
|
||||
listOf(
|
||||
BasicNameValuePair(
|
||||
@@ -61,17 +58,17 @@ internal class AcidimgHost(
|
||||
throw HostException(e)
|
||||
}
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
|
||||
XpathUtils.getAsNode(doc, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
} ?: throw HostException(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'", IMG_XPATH, url
|
||||
"Xpath '%s' cannot be found in '%s'", IMG_XPATH, image.url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
log.debug(String.format("Resolving name and image url for %s", image.url))
|
||||
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim()
|
||||
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim()
|
||||
Pair(
|
||||
|
||||
@@ -1,32 +1,25 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import java.util.*
|
||||
|
||||
internal class DPicMeHost(
|
||||
httpService: HTTPService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host("dpic.me", 1, httpService, dataTransaction, downloadSpeedService) {
|
||||
internal class DPicMeHost : Host("dpic.me", 1) {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
image: ImageEntity,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val document = fetchDocument(image.url, context)
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
|
||||
XpathUtils.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
@@ -34,11 +27,11 @@ internal class DPicMeHost(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
image.url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
log.debug(String.format("Resolving name and image url for %s", image.url))
|
||||
val imgTitle =
|
||||
Optional.ofNullable(imgNode.attributes.getNamedItem("alt"))
|
||||
.map { e: Node -> e.textContent.trim() }
|
||||
|
||||
@@ -4,6 +4,7 @@ import kotlinx.coroutines.cancelAndJoin
|
||||
import kotlinx.coroutines.delay
|
||||
import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.exception.DownloadException
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.services.DataTransaction
|
||||
@@ -18,6 +19,8 @@ import org.apache.hc.client5.http.classic.methods.HttpHead
|
||||
import org.apache.hc.core5.http.ClassicHttpResponse
|
||||
import org.apache.hc.core5.http.Header
|
||||
import org.jetbrains.exposed.sql.transactions.transaction
|
||||
import org.koin.core.component.KoinComponent
|
||||
import org.koin.core.component.inject
|
||||
import org.w3c.dom.Document
|
||||
import java.io.BufferedOutputStream
|
||||
import java.nio.file.Files
|
||||
@@ -26,12 +29,13 @@ import java.nio.file.Path
|
||||
internal abstract class Host(
|
||||
val hostName: String,
|
||||
val hostId: Byte,
|
||||
private val httpService: HTTPService,
|
||||
private val dataTransaction: DataTransaction,
|
||||
private val downloadSpeedService: DownloadSpeedService
|
||||
) {
|
||||
) : KoinComponent {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
private val httpService: HTTPService by inject()
|
||||
private val dataTransaction: DataTransaction by inject()
|
||||
private val downloadSpeedService: DownloadSpeedService by inject()
|
||||
|
||||
companion object {
|
||||
private const val READ_BUFFER_SIZE = 8192
|
||||
private val hosts: MutableMap<String, Byte> = mutableMapOf()
|
||||
@@ -47,49 +51,49 @@ internal abstract class Host(
|
||||
|
||||
@Throws(HostException::class)
|
||||
abstract fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
image: ImageEntity,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String>
|
||||
|
||||
@Throws(HostException::class)
|
||||
fun downloadInternal(url: String, context: ImageDownloadContext): DownloadedImage {
|
||||
val headers = head(url, context)
|
||||
fun downloadInternal(image: ImageEntity, context: ImageDownloadContext): DownloadedImage {
|
||||
if (hostId == 8.toByte()) {
|
||||
return downloadByHost(image, context)
|
||||
}
|
||||
val headers = head(image.url, context)
|
||||
// is the body of type image ?
|
||||
val imageMimeType = getImageMimeType(headers)
|
||||
val downloadedImage = if (imageMimeType != null) {
|
||||
// a direct link, awesome
|
||||
val downloadedImage = fetch(url, context) {
|
||||
val downloadedImage = fetch(image.url, context) {
|
||||
handleImageDownload(it, context)
|
||||
}
|
||||
DownloadedImage(getDefaultImageName(url), downloadedImage.first, downloadedImage.second)
|
||||
DownloadedImage(getDefaultImageName(image.url), downloadedImage.first, downloadedImage.second)
|
||||
} else {
|
||||
// linked image ?
|
||||
val value = headers.find { it.name.contains("content-type", true) }?.value
|
||||
if (value != null) {
|
||||
if (value.contains("text/html")) {
|
||||
val document = fetch(url, context) {
|
||||
HtmlUtils.clean(it.entity.content)
|
||||
}
|
||||
if (log.isDebugEnabled) {
|
||||
log.debug("Cleaning $url response", url)
|
||||
}
|
||||
val resolvedImage = resolve(url, document, context)
|
||||
val downloadImage: Pair<Path, ImageMimeType> =
|
||||
fetch(resolvedImage.second, context) {
|
||||
handleImageDownload(it, context)
|
||||
}
|
||||
DownloadedImage(resolvedImage.first, downloadImage.first, downloadImage.second)
|
||||
downloadByHost(image, context)
|
||||
} else {
|
||||
throw HostException("Unable to download $url, can't process content type $value")
|
||||
throw HostException("Unable to download ${image.url}, can't process content type $value")
|
||||
}
|
||||
} else {
|
||||
throw HostException("Unexpected server response for $url, response have no content type")
|
||||
throw HostException("Unexpected server response for ${image.url}, response have no content type")
|
||||
}
|
||||
}
|
||||
return downloadedImage
|
||||
}
|
||||
|
||||
private fun downloadByHost(image: ImageEntity, context: ImageDownloadContext): DownloadedImage {
|
||||
val resolvedImage = resolve(image, context)
|
||||
val downloadImage: Pair<Path, ImageMimeType> =
|
||||
fetch(resolvedImage.second, context) {
|
||||
handleImageDownload(it, context)
|
||||
}
|
||||
return DownloadedImage(resolvedImage.first, downloadImage.first, downloadImage.second)
|
||||
}
|
||||
|
||||
private fun handleImageDownload(
|
||||
response: ClassicHttpResponse,
|
||||
context: ImageDownloadContext
|
||||
@@ -183,6 +187,19 @@ internal abstract class Host(
|
||||
}
|
||||
}
|
||||
|
||||
fun fetchDocument(
|
||||
url: String,
|
||||
context: ImageDownloadContext
|
||||
): Document {
|
||||
return fetch(url, context) {
|
||||
HtmlUtils.clean(it.entity.content)
|
||||
}.also {
|
||||
if (log.isDebugEnabled) {
|
||||
log.debug("Cleaning $url response", url)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@OptIn(ExperimentalStdlibApi::class)
|
||||
private fun getImageMimeType(headers: Array<Header>): ImageMimeType? {
|
||||
|
||||
|
||||
@@ -1,37 +1,30 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.HtmlUtils
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.apache.hc.client5.http.impl.cookie.BasicClientCookie
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import java.sql.Date
|
||||
import java.time.LocalDateTime
|
||||
import java.time.ZoneId
|
||||
import java.util.*
|
||||
|
||||
internal class ImageBamHost(
|
||||
httpService: HTTPService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host("imagebam.com", 2, httpService, dataTransaction, downloadSpeedService) {
|
||||
internal class ImageBamHost : Host("imagebam.com", 2) {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
image: ImageEntity,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val document = fetchDocument(image.url, context)
|
||||
val doc = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_XPATH, url))
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_XPATH, image.url))
|
||||
if (XpathUtils.getAsNode(document, CONTINUE_XPATH) != null) {
|
||||
val clientCookie = BasicClientCookie("nsfw_inter", "1")
|
||||
clientCookie.domain = "www.imagebam.com"
|
||||
@@ -41,7 +34,7 @@ internal class ImageBamHost(
|
||||
LocalDateTime.now().plusDays(3).atZone(ZoneId.systemDefault()).toInstant()
|
||||
)
|
||||
context.httpContext.cookieStore.addCookie(clientCookie)
|
||||
fetch(url, context) {
|
||||
fetch(image.url, context) {
|
||||
HtmlUtils.clean(it.entity.content)
|
||||
}
|
||||
} else {
|
||||
@@ -51,7 +44,7 @@ internal class ImageBamHost(
|
||||
throw HostException(e)
|
||||
}
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
|
||||
XpathUtils.getAsNode(doc, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
@@ -59,11 +52,11 @@ internal class ImageBamHost(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
image.url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
log.debug(String.format("Resolving name and image url for %s", image.url))
|
||||
val imgTitle = Optional.ofNullable(imgNode.attributes.getNamedItem("alt"))
|
||||
.map { e: Node -> e.textContent.trim { it <= ' ' } }
|
||||
.orElse("")
|
||||
|
||||
@@ -1,32 +1,25 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import java.util.*
|
||||
|
||||
internal class ImageTwistHost(
|
||||
httpService: HTTPService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host("imagetwist.com", 3, httpService, dataTransaction, downloadSpeedService) {
|
||||
internal class ImageTwistHost : Host("imagetwist.com", 3) {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
image: ImageEntity,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val document = fetchDocument(image.url, context)
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
|
||||
XpathUtils.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
@@ -34,11 +27,11 @@ internal class ImageTwistHost(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
image.url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
log.debug(String.format("Resolving name and image url for %s", image.url))
|
||||
val imgTitle =
|
||||
Optional.ofNullable(imgNode.attributes.getNamedItem("alt"))
|
||||
.map { obj: Node -> obj.textContent }
|
||||
|
||||
@@ -1,41 +1,34 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.HtmlUtils
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
|
||||
internal class ImageVenueHost(
|
||||
httpService: HTTPService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host("imagevenue.com", 4, httpService, dataTransaction, downloadSpeedService) {
|
||||
internal class ImageVenueHost : Host("imagevenue.com", 4) {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
image: ImageEntity,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val doc = try {
|
||||
val document = fetchDocument(image.url, context)
|
||||
log.debug(
|
||||
String.format(
|
||||
"Looking for xpath expression %s in %s",
|
||||
CONTINUE_BUTTON_XPATH,
|
||||
url
|
||||
image.url
|
||||
)
|
||||
)
|
||||
if (XpathUtils.getAsNode(document, CONTINUE_BUTTON_XPATH) != null) {
|
||||
// Button detected. No need to actually click it, just make the call again.
|
||||
fetch(url, context) {
|
||||
fetch(image.url, context) {
|
||||
HtmlUtils.clean(it.entity.content)
|
||||
}
|
||||
} else {
|
||||
@@ -45,7 +38,7 @@ internal class ImageVenueHost(
|
||||
throw HostException(e)
|
||||
}
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
|
||||
XpathUtils.getAsNode(doc, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
@@ -53,11 +46,11 @@ internal class ImageVenueHost(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
image.url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
log.debug(String.format("Resolving name and image url for %s", image.url))
|
||||
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' }
|
||||
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
|
||||
Pair(
|
||||
|
||||
@@ -1,30 +1,23 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
|
||||
internal class ImageZillaHost(
|
||||
httpService: HTTPService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host("imagezilla.net", 5, httpService, dataTransaction, downloadSpeedService) {
|
||||
internal class ImageZillaHost : Host("imagezilla.net", 5) {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
image: ImageEntity,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val document = fetchDocument(image.url, context)
|
||||
val titleNode = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
|
||||
XpathUtils.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
@@ -32,17 +25,17 @@ internal class ImageZillaHost(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
image.url
|
||||
)
|
||||
)
|
||||
log.debug(String.format("Resolving name for %s", url))
|
||||
log.debug(String.format("Resolving name for %s", image.url))
|
||||
var title = titleNode.attributes.getNamedItem("title").textContent.trim()
|
||||
titleNode.textContent.trim()
|
||||
if (title.isEmpty()) {
|
||||
title = getDefaultImageName(url)
|
||||
title = getDefaultImageName(image.url)
|
||||
}
|
||||
return try {
|
||||
Pair(title, url.replace("show", "images"))
|
||||
Pair(title, image.url.replace("show", "images"))
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Unexpected error occurred", e)
|
||||
}
|
||||
|
||||
@@ -1,31 +1,24 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
|
||||
internal class ImgSpiceHost(
|
||||
httpService: HTTPService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host("imgspice.com", 7, httpService, dataTransaction, downloadSpeedService) {
|
||||
internal class ImgSpiceHost : Host("imgspice.com", 7) {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
image: ImageEntity,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val document = fetchDocument(image.url, context)
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
|
||||
XpathUtils.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
@@ -33,11 +26,11 @@ internal class ImgSpiceHost(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
image.url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
log.debug(String.format("Resolving name and image url for %s", image.url))
|
||||
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' }
|
||||
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
|
||||
Pair(
|
||||
|
||||
@@ -1,31 +1,24 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
|
||||
internal class ImgboxHost(
|
||||
httpService: HTTPService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host("imgbox.com", 6, httpService, dataTransaction, downloadSpeedService) {
|
||||
internal class ImgboxHost : Host("imgbox.com", 6) {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
image: ImageEntity,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val document = fetchDocument(image.url, context)
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
|
||||
XpathUtils.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
@@ -33,11 +26,11 @@ internal class ImgboxHost(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
image.url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
log.debug(String.format("Resolving name and image url for %s", image.url))
|
||||
val imgTitle = imgNode.attributes.getNamedItem("title").textContent.trim { it <= ' ' }
|
||||
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
|
||||
Pair(imgTitle, imgUrl)
|
||||
|
||||
@@ -1,90 +1,79 @@
|
||||
package me.vripper.host
|
||||
|
||||
import com.github.benmanes.caffeine.cache.Cache
|
||||
import com.github.benmanes.caffeine.cache.Caffeine
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.HtmlProcessorException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.HtmlUtils
|
||||
import me.vripper.utilities.ApplicationProperties.IMX_SUBDOMAINS
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.apache.hc.client5.http.classic.methods.HttpPost
|
||||
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
|
||||
import org.apache.hc.core5.http.message.BasicNameValuePair
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import java.io.IOException
|
||||
import me.vripper.utilities.RequestLimit
|
||||
import org.apache.hc.client5.http.classic.methods.HttpHead
|
||||
import org.apache.hc.client5.http.config.ConnectionConfig
|
||||
import org.apache.hc.client5.http.impl.classic.HttpClients
|
||||
import org.apache.hc.client5.http.impl.io.BasicHttpClientConnectionManager
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
|
||||
internal class ImxHost : Host("imx.to", 8) {
|
||||
companion object {
|
||||
private val resolvedHosts: Cache<Long, String> =
|
||||
Caffeine.newBuilder().expireAfterAccess(5, TimeUnit.MINUTES).build()
|
||||
}
|
||||
|
||||
internal class ImxHost(
|
||||
private val httpService: HTTPService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host("imx.to", 8, httpService, dataTransaction, downloadSpeedService) {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
private val failFastHttpClient = HttpClients.custom().apply {
|
||||
setConnectionManager(BasicHttpClientConnectionManager().apply {
|
||||
connectionConfig = ConnectionConfig.custom()
|
||||
.setConnectTimeout(5000, TimeUnit.MILLISECONDS)
|
||||
.setSocketTimeout(5000, TimeUnit.MILLISECONDS)
|
||||
.build()
|
||||
})
|
||||
}.build()
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
image: ImageEntity,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val httpsUrl = url.replace("http:", "https:")
|
||||
var value: String? = null
|
||||
try {
|
||||
log.debug("Looking for xpath expression $CONTINUE_BUTTON_XPATH in $httpsUrl")
|
||||
val contDiv = XpathUtils.getAsNode(document, CONTINUE_BUTTON_XPATH)
|
||||
?: throw HostException("$CONTINUE_BUTTON_XPATH cannot be found")
|
||||
val node = contDiv.attributes.getNamedItem("value")
|
||||
if (node != null) {
|
||||
value = node.textContent
|
||||
log.debug("Resolving name and image url for ${image.url}")
|
||||
val imgTitle = String.format("IMG_%04d", image.index + 1)
|
||||
synchronized(image.postId.toString().intern()) {
|
||||
val subDomain = resolvedHosts.getIfPresent(image.postId)
|
||||
if (subDomain != null) {
|
||||
val imgUrl = findPattern(image, subDomain)
|
||||
return Pair(
|
||||
imgTitle.ifEmpty { getDefaultImageName(imgUrl) }, imgUrl
|
||||
)
|
||||
} else {
|
||||
IMX_SUBDOMAINS.forEach { subDomain ->
|
||||
val imgUrl = findPattern(image, subDomain)
|
||||
val result = runCatching {
|
||||
RequestLimit.getPermit(1)
|
||||
val httpHead = HttpHead(imgUrl).also { context.requests.add(it) }
|
||||
failFastHttpClient.execute(httpHead) { response ->
|
||||
if (response.code / 100 != 2) {
|
||||
throw HostException("Invalid response code ${response.code}")
|
||||
}
|
||||
}
|
||||
}
|
||||
if (result.isSuccess) {
|
||||
resolvedHosts.put(image.postId, subDomain)
|
||||
return Pair(
|
||||
imgTitle.ifEmpty { getDefaultImageName(imgUrl) }, imgUrl
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
}
|
||||
if (value == null) {
|
||||
throw HostException("Failed to obtain value attribute from continue input")
|
||||
}
|
||||
log.debug("Click button found for $httpsUrl")
|
||||
val httpPost: HttpPost = HttpPost(httpsUrl).also {
|
||||
it.entity = UrlEncodedFormEntity(listOf(BasicNameValuePair("imgContinue", value)))
|
||||
}.also { context.requests.add(it) }
|
||||
log.debug("Requesting {}", httpPost)
|
||||
val doc = try {
|
||||
httpService.client.execute(
|
||||
httpPost, context.httpContext
|
||||
) { response ->
|
||||
log.debug("Cleaning response for {}", httpPost)
|
||||
HtmlUtils.clean(response.entity.content)
|
||||
}
|
||||
} catch (e: IOException) {
|
||||
throw HostException(e)
|
||||
} catch (e: HtmlProcessorException) {
|
||||
throw HostException(e)
|
||||
}
|
||||
val imgNode: Node = try {
|
||||
log.debug("Looking for xpath expression $IMG_XPATH in $httpsUrl")
|
||||
XpathUtils.getAsNode(doc, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
} ?: throw HostException(
|
||||
"Xpath $IMG_XPATH cannot be found in $httpsUrl"
|
||||
)
|
||||
return try {
|
||||
log.debug("Resolving name and image url for $httpsUrl")
|
||||
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' }
|
||||
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
|
||||
Pair(
|
||||
imgTitle.ifEmpty { getDefaultImageName(imgUrl) }, imgUrl
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Unexpected error occurred", e)
|
||||
}
|
||||
throw HostException("Unable to find full size image for ${image.url}")
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val CONTINUE_BUTTON_XPATH = "//*[@name='imgContinue']"
|
||||
private const val IMG_XPATH = "//img[@class='centred']"
|
||||
}
|
||||
private fun findPattern(image: ImageEntity, subDomain: String): String = image.thumbUrl
|
||||
.replace("http:", "https:")
|
||||
.replace("imx.to", subDomain)
|
||||
.replace("upload/small/", "i/")
|
||||
.replace("u/t/", "i/")
|
||||
.replace("t/", "i/")
|
||||
}
|
||||
@@ -1,35 +1,30 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.HtmlUtils
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import java.net.URI
|
||||
import java.net.URISyntaxException
|
||||
|
||||
internal class PimpandhostHost(
|
||||
httpService: HTTPService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host("pimpandhost.com", 9, httpService, dataTransaction, downloadSpeedService) {
|
||||
internal class PimpandhostHost : Host("pimpandhost.com", 9) {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
image: ImageEntity,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val newUrl: String
|
||||
try {
|
||||
newUrl = appendUri(url.replace("http://", "https://").replace("-medium(\\.html)?".toRegex(), ""), "size=original")
|
||||
newUrl = appendUri(
|
||||
image.url.replace("http://", "https://").replace("-medium(\\.html)?".toRegex(), ""),
|
||||
"size=original"
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
throw HostException(e)
|
||||
}
|
||||
@@ -45,7 +40,7 @@ internal class PimpandhostHost(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
image.url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
|
||||
@@ -1,31 +1,24 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
|
||||
internal class PixRouteHost(
|
||||
httpService: HTTPService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host("pixroute.com", 11, httpService, dataTransaction, downloadSpeedService) {
|
||||
internal class PixRouteHost : Host("pixroute.com", 11) {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
image: ImageEntity,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val document = fetchDocument(image.url, context)
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
|
||||
XpathUtils.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
@@ -33,11 +26,11 @@ internal class PixRouteHost(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
image.url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
log.debug(String.format("Resolving name and image url for %s", image.url))
|
||||
Pair(imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' },
|
||||
imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' })
|
||||
} catch (e: Exception) {
|
||||
|
||||
@@ -1,31 +1,24 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
|
||||
internal class PixhostHost(
|
||||
httpService: HTTPService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host("pixhost.to", 10, httpService, dataTransaction, downloadSpeedService) {
|
||||
internal class PixhostHost : Host("pixhost.to", 10) {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
image: ImageEntity,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val document = fetchDocument(image.url, context)
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
|
||||
XpathUtils.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
@@ -33,11 +26,11 @@ internal class PixhostHost(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
image.url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
log.debug(String.format("Resolving name and image url for %s", image.url))
|
||||
val imgTitle = imgNode.attributes.getNamedItem("alt").textContent.trim { it <= ' ' }
|
||||
val imgUrl = imgNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' }
|
||||
Pair(imgTitle.substring(imgTitle.indexOf('_') + 1), imgUrl)
|
||||
|
||||
@@ -1,30 +1,23 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
|
||||
internal class PixxxelsHost(
|
||||
httpService: HTTPService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host("pixxxels.cc", 12, httpService, dataTransaction, downloadSpeedService) {
|
||||
internal class PixxxelsHost : Host("pixxxels.cc", 12) {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
image: ImageEntity,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val document = fetchDocument(image.url, context)
|
||||
val imgNode = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
|
||||
XpathUtils.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
@@ -32,11 +25,11 @@ internal class PixxxelsHost(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
image.url
|
||||
)
|
||||
)
|
||||
val titleNode = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url))
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, image.url))
|
||||
XpathUtils.getAsNode(document, TITLE_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
@@ -44,11 +37,11 @@ internal class PixxxelsHost(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
TITLE_XPATH,
|
||||
url
|
||||
image.url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
log.debug(String.format("Resolving name and image url for %s", image.url))
|
||||
val imgTitle = titleNode.textContent.trim { it <= ' ' }
|
||||
val imgUrl = imgNode.attributes.getNamedItem("href").textContent.trim { it <= ' ' }
|
||||
Pair(
|
||||
|
||||
@@ -1,32 +1,25 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import java.util.*
|
||||
|
||||
internal class PostImgHost(
|
||||
httpService: HTTPService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host("postimg.cc", 13, httpService, dataTransaction, downloadSpeedService) {
|
||||
internal class PostImgHost : Host("postimg.cc", 13) {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
image: ImageEntity,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val document = fetchDocument(image.url, context)
|
||||
val titleNode = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url))
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, image.url))
|
||||
XpathUtils.getAsNode(document, TITLE_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
@@ -34,11 +27,11 @@ internal class PostImgHost(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
TITLE_XPATH,
|
||||
url
|
||||
image.url
|
||||
)
|
||||
)
|
||||
val urlNode = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
|
||||
XpathUtils.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
@@ -46,14 +39,14 @@ internal class PostImgHost(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
image.url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
log.debug(String.format("Resolving name and image url for %s", image.url))
|
||||
val imgTitle = Optional.ofNullable(titleNode)
|
||||
.map { node: Node -> node.textContent.trim { it <= ' ' } }
|
||||
.orElseGet { getDefaultImageName(url) }
|
||||
.orElseGet { getDefaultImageName(image.url) }
|
||||
Pair(imgTitle, urlNode.attributes.getNamedItem("href").textContent.trim { it <= ' ' })
|
||||
} catch (e: Exception) {
|
||||
throw HostException("Unexpected error occurred", e)
|
||||
|
||||
@@ -1,47 +1,40 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
|
||||
internal class TurboImageHost(
|
||||
httpService: HTTPService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host("turboimagehost.com", 14, httpService, dataTransaction, downloadSpeedService) {
|
||||
internal class TurboImageHost : Host("turboimagehost.com", 14) {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
image: ImageEntity,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val document = fetchDocument(image.url, context)
|
||||
var title: String?
|
||||
title = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, url))
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", TITLE_XPATH, image.url))
|
||||
val titleNode: Node? = XpathUtils.getAsNode(document, TITLE_XPATH)
|
||||
log.debug(String.format("Resolving name for %s", url))
|
||||
log.debug(String.format("Resolving name for %s", image.url))
|
||||
titleNode?.textContent?.trim { it <= ' ' }
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
}
|
||||
if (title.isNullOrEmpty()) {
|
||||
title = getDefaultImageName(url)
|
||||
title = getDefaultImageName(image.url)
|
||||
}
|
||||
val urlNode: Node = XpathUtils.getAsNode(document, IMG_XPATH)
|
||||
?: throw HostException(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
image.url
|
||||
)
|
||||
)
|
||||
return Pair(title, urlNode.attributes.getNamedItem("src").textContent.trim { it <= ' ' })
|
||||
|
||||
@@ -1,32 +1,25 @@
|
||||
package me.vripper.host
|
||||
|
||||
import me.vripper.entities.ImageEntity
|
||||
import me.vripper.exception.HostException
|
||||
import me.vripper.exception.XpathException
|
||||
import me.vripper.services.DataTransaction
|
||||
import me.vripper.services.DownloadService.ImageDownloadContext
|
||||
import me.vripper.services.DownloadSpeedService
|
||||
import me.vripper.services.HTTPService
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.XpathUtils
|
||||
import org.w3c.dom.Document
|
||||
import org.w3c.dom.Node
|
||||
import java.util.*
|
||||
|
||||
internal class ViprImHost(
|
||||
httpService: HTTPService,
|
||||
dataTransaction: DataTransaction,
|
||||
downloadSpeedService: DownloadSpeedService,
|
||||
) : Host("vipr.im", 15, httpService, dataTransaction, downloadSpeedService) {
|
||||
internal class ViprImHost : Host("vipr.im", 15) {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
@Throws(HostException::class)
|
||||
override fun resolve(
|
||||
url: String,
|
||||
document: Document,
|
||||
image: ImageEntity,
|
||||
context: ImageDownloadContext
|
||||
): Pair<String, String> {
|
||||
val document = fetchDocument(image.url, context)
|
||||
val imgNode: Node = try {
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, url))
|
||||
log.debug(String.format("Looking for xpath expression %s in %s", IMG_XPATH, image.url))
|
||||
XpathUtils.getAsNode(document, IMG_XPATH)
|
||||
} catch (e: XpathException) {
|
||||
throw HostException(e)
|
||||
@@ -34,11 +27,11 @@ internal class ViprImHost(
|
||||
String.format(
|
||||
"Xpath '%s' cannot be found in '%s'",
|
||||
IMG_XPATH,
|
||||
url
|
||||
image.url
|
||||
)
|
||||
)
|
||||
return try {
|
||||
log.debug(String.format("Resolving name and image url for %s", url))
|
||||
log.debug(String.format("Resolving name and image url for %s", image.url))
|
||||
val imgTitle =
|
||||
Optional.ofNullable(imgNode.attributes.getNamedItem("alt"))
|
||||
.map { obj: Node -> obj.textContent }
|
||||
|
||||
@@ -19,13 +19,15 @@ import me.vripper.host.ImageMimeType
|
||||
import me.vripper.model.ErrorCount
|
||||
import me.vripper.model.QueueState
|
||||
import me.vripper.model.Settings
|
||||
import me.vripper.utilities.ApplicationProperties
|
||||
import me.vripper.utilities.LoggerDelegate
|
||||
import me.vripper.utilities.PathUtils.getExtension
|
||||
import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
|
||||
import me.vripper.utilities.PathUtils.sanitize
|
||||
import me.vripper.utilities.downloadRunner
|
||||
import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase
|
||||
import org.apache.hc.client5.http.cookie.BasicCookieStore
|
||||
import org.apache.hc.client5.http.cookie.Cookie
|
||||
import org.apache.hc.client5.http.impl.cookie.BasicClientCookie
|
||||
import org.apache.hc.client5.http.protocol.HttpClientContext
|
||||
import org.jetbrains.exposed.sql.transactions.transaction
|
||||
import org.koin.core.component.KoinComponent
|
||||
@@ -34,11 +36,14 @@ import java.io.IOException
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.StandardCopyOption
|
||||
import java.time.Instant
|
||||
import java.util.*
|
||||
import java.util.concurrent.locks.ReentrantLock
|
||||
import kotlin.concurrent.withLock
|
||||
import kotlin.io.path.Path
|
||||
import kotlin.io.path.listDirectoryEntries
|
||||
import kotlin.io.path.pathString
|
||||
import kotlin.io.path.readLines
|
||||
|
||||
internal class DownloadService(
|
||||
private val settingsService: SettingsService,
|
||||
@@ -46,6 +51,7 @@ internal class DownloadService(
|
||||
private val retryPolicyService: RetryPolicyService,
|
||||
private val eventBus: EventBus
|
||||
) {
|
||||
|
||||
private val maxPoolSize: Int = 24
|
||||
private val log by LoggerDelegate()
|
||||
private val running: MutableMap<Byte, MutableList<ImageDownloadRunnable>> = mutableMapOf()
|
||||
@@ -56,10 +62,52 @@ internal class DownloadService(
|
||||
private var downloadMonitorThread: Thread? = null
|
||||
|
||||
internal class ImageDownloadContext(val imageEntity: ImageEntity, val settings: Settings) : KoinComponent {
|
||||
private val log by LoggerDelegate()
|
||||
|
||||
init {
|
||||
ApplicationProperties.VRIPPER_DIR
|
||||
.listDirectoryEntries()
|
||||
.filter { it.fileName.pathString.startsWith("cookies") }
|
||||
.forEach { cookiesPath ->
|
||||
loadCookies(cookiesPath).also { cookies ->
|
||||
cookies.forEach { cookie ->
|
||||
if (HTTPService.cookieStore.cookies.find { it.name == cookie.name } == null) {
|
||||
log.info("Applying cookie: ${cookie.name}")
|
||||
HTTPService.cookieStore.addCookie(cookie)
|
||||
} else {
|
||||
log.warn("Cookie already loaded: ${cookie.name}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||
private val jobs = mutableListOf<Job>()
|
||||
val httpContext: HttpClientContext =
|
||||
HttpClientContext.create().apply { cookieStore = BasicCookieStore() }
|
||||
val httpContext: HttpClientContext = HttpClientContext.create().apply {
|
||||
cookieStore = HTTPService.cookieStore
|
||||
}
|
||||
|
||||
|
||||
private fun loadCookies(cookiesPath: Path): List<Cookie> {
|
||||
return cookiesPath.readLines()
|
||||
.map { it.trim() }
|
||||
.filter { it.isNotBlank() }
|
||||
.filter { !it.startsWith("#") }
|
||||
.map { line ->
|
||||
val cookieComponents = line.split("\t")
|
||||
BasicClientCookie(cookieComponents[5], cookieComponents[6]).apply {
|
||||
domain = cookieComponents[0]
|
||||
isHttpOnly = cookieComponents[1].toBoolean()
|
||||
path = cookieComponents[2]
|
||||
isSecure = cookieComponents[3].toBoolean()
|
||||
setExpiryDate(Instant.ofEpochSecond(cookieComponents[4].toLong()))
|
||||
}
|
||||
}.also {
|
||||
log.info("Found ${it.size} cookies in $cookiesPath")
|
||||
}
|
||||
}
|
||||
|
||||
val requests = mutableListOf<HttpUriRequestBase>()
|
||||
val postId = imageEntity.postIdRef
|
||||
|
||||
@@ -102,7 +150,7 @@ internal class DownloadService(
|
||||
}
|
||||
log.debug("Getting image url and name from ${imageEntity.url} using ${imageEntity.host}")
|
||||
val host = hosts.first { it.isSupported(imageEntity.url) }
|
||||
val downloadedImage = host.downloadInternal(imageEntity.url, context)
|
||||
val downloadedImage = host.downloadInternal(imageEntity, context)
|
||||
log.debug("Resolved name for ${imageEntity.url}: ${downloadedImage.name}")
|
||||
log.debug("Downloaded image {} to {}", imageEntity.url, downloadedImage.path)
|
||||
synchronized(imageEntity.postId.toString().intern()) {
|
||||
|
||||
@@ -4,6 +4,7 @@ import kotlinx.coroutines.*
|
||||
import me.vripper.utilities.ApplicationProperties
|
||||
import org.apache.hc.client5.http.config.ConnectionConfig
|
||||
import org.apache.hc.client5.http.config.RequestConfig
|
||||
import org.apache.hc.client5.http.cookie.BasicCookieStore
|
||||
import org.apache.hc.client5.http.cookie.StandardCookieSpec
|
||||
import org.apache.hc.client5.http.impl.DefaultRedirectStrategy
|
||||
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient
|
||||
@@ -16,6 +17,9 @@ import org.apache.hc.core5.pool.PoolReusePolicy
|
||||
import org.apache.hc.core5.util.Timeout
|
||||
|
||||
internal class HTTPService {
|
||||
companion object {
|
||||
val cookieStore = BasicCookieStore()
|
||||
}
|
||||
private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
||||
|
||||
private var pcm: HttpClientConnectionManager = BasicHttpClientConnectionManager()
|
||||
@@ -64,6 +68,7 @@ internal class HTTPService {
|
||||
.setUserAgent(ApplicationProperties.USER_AGENT)
|
||||
.disableAutomaticRetries()
|
||||
.setDefaultRequestConfig(rc)
|
||||
.setDefaultCookieStore(cookieStore)
|
||||
.build()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,10 @@ object ApplicationProperties {
|
||||
ignoreUnknownKeys = true
|
||||
}
|
||||
const val DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:136.0) Gecko/20100101 Firefox/136.0"
|
||||
val USER_AGENT: String;
|
||||
val USER_AGENT: String
|
||||
|
||||
const val DEFAULT_IMX_SUBDOMAINS = "i.imx.to,i001.imx.to,i002.imx.to,i004.imx.to,i006.imx.to,i003.imx.to"
|
||||
val IMX_SUBDOMAINS: Set<String>
|
||||
|
||||
@Serializable
|
||||
internal data class ReleaseResponse(@SerialName("tag_name") val tagName: String)
|
||||
@@ -35,12 +38,19 @@ object ApplicationProperties {
|
||||
init {
|
||||
Files.createDirectories(VRIPPER_DIR)
|
||||
System.setProperty("VRIPPER_DIR", VRIPPER_DIR.toRealPath().pathString)
|
||||
val userAgentOverride = VRIPPER_DIR.resolve("user-agent")
|
||||
val userAgentOverride = VRIPPER_DIR.resolve("user-agent.txt")
|
||||
USER_AGENT = if (userAgentOverride.exists()) {
|
||||
userAgentOverride.readLines().firstOrNull() ?: DEFAULT_USER_AGENT
|
||||
} else {
|
||||
DEFAULT_USER_AGENT
|
||||
}
|
||||
|
||||
val imxSubdomainsOverride = VRIPPER_DIR.resolve("imx-subdomains.txt")
|
||||
IMX_SUBDOMAINS = if (imxSubdomainsOverride.exists()) {
|
||||
imxSubdomainsOverride.readLines().firstOrNull() ?: DEFAULT_IMX_SUBDOMAINS
|
||||
} else {
|
||||
DEFAULT_IMX_SUBDOMAINS
|
||||
}.split(",").toSet()
|
||||
}
|
||||
|
||||
fun latestVersion(): String {
|
||||
|
||||
Reference in New Issue
Block a user