fixes #231 (#232)

.github/workflows/release.yml

@@ -8,12 +8,12 @@ jobs:
   build:
     strategy:
       matrix:
-        os: [ ubuntu-latest, windows-latest, macos-latest ]
+        os: [ ubuntu-latest, windows-latest, macos-13, macos-latest ]
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v3
 
-    - name: Set up JDK 21
+    - name: Set up jdk21
       uses: actions/setup-java@v3
       with:
         java-version: '21'
@@ -21,37 +21,29 @@ jobs:
         server-id: github # Value of the distributionManagement/repository/id field of the pom.xml
         settings-path: ${{ github.workspace }} # location for the settings.xml file
 
-    - name: Build parent project
-      run: mvn -N -q install --file pom.xml
-
-    - name: Build Core Jar
-      run: mvn -B -q install --file vripper-core/pom.xml
-
-    - name: Build GUI Jar
-      run: mvn -B -q install --file vripper-gui/pom.xml
-
-    - name: Rename GUI Jar
-      run: mv vripper-gui/target/vripper-gui-${{ github.event.release.tag_name }}-jar-with-dependencies.jar vripper-gui/target/vripper-noarch-gui-${{ github.event.release.tag_name }}.jar
+    - name: Build gui jar
+      run: |
+        mvn -N -q install --file pom.xml
+        mvn -B -q install --file vripper-core/pom.xml
+        mvn -B -q install --file vripper-gui/pom.xml
+        mv vripper-gui/target/vripper-gui-${{ github.event.release.tag_name }}-jar-with-dependencies.jar vripper-gui/target/vripper-noarch-gui-${{ github.event.release.tag_name }}.jar
 
     - if: matrix.os == 'ubuntu-latest'
-      name: Release GUI Jar
+      name: Release gui Jar
       uses: softprops/action-gh-release@v1
       with:
         files: vripper-gui/target/vripper-noarch-gui-${{ github.event.release.tag_name }}.jar
 
     # Start building WEB jar in ubuntu only
     - if: matrix.os == 'ubuntu-latest'
-      name: Build WEB Jar
+      name: Build web Jar
       run: |
         mvn -B -q install --file vripper-web-ui/pom.xml
         mvn -B -q install --file vripper-web/pom.xml
+        mv vripper-web/target/vripper-web-${{ github.event.release.tag_name }}.jar vripper-web/target/vripper-noarch-web-${{ github.event.release.tag_name }}.jar
 
     - if: matrix.os == 'ubuntu-latest'
-      name: Rename WEB Jar
-      run: mv vripper-web/target/vripper-web-${{ github.event.release.tag_name }}.jar vripper-web/target/vripper-noarch-web-${{ github.event.release.tag_name }}.jar
-
-    - if: matrix.os == 'ubuntu-latest'
-      name: Release WEB Jar
+      name: Release web Jar
       uses: softprops/action-gh-release@v1
       with:
         files: vripper-web/target/vripper-noarch-web-${{ github.event.release.tag_name }}.jar
@@ -62,7 +54,7 @@ jobs:
         cp vripper-gui/target/vripper-noarch-gui-${{ github.event.release.tag_name }}.jar jpackage/jar/vripper-gui.jar
         
     - if: matrix.os == 'ubuntu-latest'
-      name: Package Linux
+      name: Package for Linux
       run: |
         cd jpackage
         jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.png
@@ -72,7 +64,7 @@ jobs:
         mv dist/vripper_${{ github.event.release.tag_name }}-1_amd64.deb dist/vripper-linux-${{ github.event.release.tag_name }}_amd64.deb
 
     - if: matrix.os == 'windows-latest'
-      name: Package Windows
+      name: Package for Windows
       run: |
         cd jpackage
         jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.ico
@@ -83,14 +75,24 @@ jobs:
         ren VRipper-${{ github.event.release.tag_name }}.exe vripper-windows-installer-${{ github.event.release.tag_name }}.exe
 
     - if: matrix.os == 'macos-latest'
-      name: Package macOS
+      name: Package for macOS(arm64)
       run: |
         cd jpackage
         jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.icns
         jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type pkg
         jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type dmg
-        mv dist/VRipper-${{ github.event.release.tag_name }}.pkg dist/vripper-macos-${{ github.event.release.tag_name }}.pkg
-        mv dist/VRipper-${{ github.event.release.tag_name }}.dmg dist/vripper-macos-${{ github.event.release.tag_name }}.dmg
+        mv dist/VRipper-${{ github.event.release.tag_name }}.pkg dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.pkg
+        mv dist/VRipper-${{ github.event.release.tag_name }}.dmg dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.dmg
+    
+    - if: matrix.os == 'macos-13'
+      name: Package for macOS(x86_64)
+      run: |
+        cd jpackage
+        jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-app-image.cfg" --icon icon.icns
+        jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type pkg
+        jpackage --app-version ${{ github.event.release.tag_name }} "@jpackage.cfg" "@jpackage-macos.cfg" --type dmg
+        mv dist/VRipper-${{ github.event.release.tag_name }}.pkg dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.pkg
+        mv dist/VRipper-${{ github.event.release.tag_name }}.dmg dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.dmg
 
     - if: matrix.os == 'ubuntu-latest'
       name: Zip Ubuntu portable
@@ -111,13 +113,22 @@ jobs:
         filename: 'vripper-windows-portable-${{ github.event.release.tag_name }}.zip'
 
     - if: matrix.os == 'macos-latest'
-      name: Zip MacOS portable
+      name: Zip macOS(arm64) portable
       uses: thedoctor0/zip-release@0.7.1
       with:
         type: 'zip'
         directory: 'jpackage/dist'
         path: 'VRipper.app'
-        filename: 'vripper-macos-portable-${{ github.event.release.tag_name }}.zip'
+        filename: 'vripper-macos-portable-${{ github.event.release.tag_name }}.arm64.zip'
+    
+    - if: matrix.os == 'macos-13'
+      name: Zip macOS(x86_64) portable
+      uses: thedoctor0/zip-release@0.7.1
+      with:
+        type: 'zip'
+        directory: 'jpackage/dist'
+        path: 'VRipper.app'
+        filename: 'vripper-macos-portable-${{ github.event.release.tag_name }}.x86_64.zip'
 
     - if: matrix.os == 'ubuntu-latest'
       name: Release packages for Linux
@@ -138,10 +149,19 @@ jobs:
           jpackage/dist/vripper-windows-portable-${{ github.event.release.tag_name }}.zip
 
     - if: matrix.os == 'macos-latest'
-      name: Release packages for macOS
+      name: Release packages for macOS(arm64)
       uses: softprops/action-gh-release@v1
       with:
         files: |
-          jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.pkg
-          jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.dmg
-          jpackage/dist/vripper-macos-portable-${{ github.event.release.tag_name }}.zip
+          jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.pkg
+          jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.arm64.dmg
+          jpackage/dist/vripper-macos-portable-${{ github.event.release.tag_name }}.arm64.zip
+          
+    - if: matrix.os == 'macos-13'
+      name: Release packages for macOS(x86_64)
+      uses: softprops/action-gh-release@v1
+      with:
+        files: |
+          jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.pkg
+          jpackage/dist/vripper-macos-${{ github.event.release.tag_name }}.x86_64.dmg
+          jpackage/dist/vripper-macos-portable-${{ github.event.release.tag_name }}.x86_64.zip

.gitignore

@@ -4,4 +4,5 @@
 /**/*/build-dir
 /**/*/java-runtime
 .idea
-.vripper
+.vripper
+.flattened-pom.xml

README.md

@@ -1,21 +1,31 @@
 # VRipper!
 
-This is my spin for a cross-platform gallery ripper for [vipergirls.to](https://vipergirls.to).
-
+This is my spin for a cross-platform gallery ripper for [vipergirls.to](https://vipergirls.to)
 
 ![GitHub Image](/image.png)
 
+## Donation
+To support this project, you can make a donation to its current maintainer
+
+[!["Buy Me A Coffee"](https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png)](https://buymeacoffee.com/devclaw)
+
+Or with Cryptocurrency
+
+ETH: 0xDdac82B16dC5E3D742fc915ffF583D8548A301cA
+
+BTC: bc1qcqudnkrndwyadsjwrxww42svkf8trnzx3c8vlr
 
 ## Requirements
 Direct access to `vipergirls.to` domain.
 
 ## Installing VRipper
 
-<img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/windows_logo.svg" width="100%" height="75"> Windows | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/linux_logo.svg" width="100%" height="75"> Linux  | <img src="https://images.vexels.com/media/users/3/166401/isolated/preview/b82aa7ac3f736dd78570dd3fa3fa9e24-java-programming-language-icon-by-vexels.png" width="100%" height="75"> Java
-:---:|:---:|:---:|:---:
-[Installer (EXE)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-windows-installer-5.8.0.exe) <br /> [Installer (MSI)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-windows-installer-5.8.0.msi) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-windows-portable-5.8.0.zip) | [Installer (DMG)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-macos-5.8.0.dmg) <br /> [Installer (PKG)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-macos-5.8.0.pkg) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-macos-portable-5.8.0.zip)  | [Linux (amd64) (DEB)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-linux-5.8.0_amd64.deb) <br /> [Linux (x86_64) (RPM)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-linux-5.8.0.x86_64.rpm) <br /> [Portable (ZIP)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-linux-portable-5.8.0.zip) | [Java GUI (noarch)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-noarch-gui-5.8.0.jar) <br /> [Java Web (noarch)](https://github.com/death-claw/vripper-project/releases/download/5.8.0/vripper-noarch-web-5.8.0.jar)
+<img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/windows_logo.svg" width="100%" height="75"> Windows | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS (Intel) | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/mac_logo.svg" width="100%" height="75"> macOS (Apple silicon) | <img src="https://github.com/stashapp/stash/raw/develop/docs/readme_assets/linux_logo.svg" width="100%" height="75"> Linux  | <img src="https://images.vexels.com/media/users/3/166401/isolated/preview/b82aa7ac3f736dd78570dd3fa3fa9e24-java-programming-language-icon-by-vexels.png" width="100%" height="75"> Java
+:---:|:---:|:---:|:---:|:---:
+[Installer (EXE)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-windows-installer-6.5.3.exe) <br /> [Installer (MSI)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-windows-installer-6.5.3.msi) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-windows-portable-6.5.3.zip) | [Installer (DMG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-macos-6.5.3.x86_64.dmg) <br /> [Installer (PKG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-macos-6.5.3.x86_64.pkg) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-macos-portable-6.5.3.x86_64.zip) | [Installer (DMG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-macos-6.5.3.arm64.dmg) <br /> [Installer (PKG)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-macos-6.5.3.arm64.pkg) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-macos-portable-6.5.3.arm64.zip)  | [Linux (amd64) (DEB)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-linux-6.5.3_amd64.deb) <br /> [Linux (x86_64) (RPM)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-linux-6.5.3.x86_64.rpm) <br /> [Portable (ZIP)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-linux-portable-6.5.3.zip) | [Java GUI (noarch)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-noarch-gui-6.5.3.jar) <br /> [Java Web (noarch)](https://github.com/dev-claw/vripper-project/releases/download/6.5.3/vripper-noarch-web-6.5.3.jar)
 
-Source code and previous versions are available on the [Releases page](https://github.com/death-claw/vripper-project/releases).  
+Source code and previous versions are available on
+the [Releases page](https://github.com/dev-claw/vripper-project/releases).
 
 Application data (application logs, settings and persisted data) is stored in:  
 * Windows --> `C:\USERS\<your Windows username>\vripper` 

jpackage/jpackage-windows.cfg

@@ -5,4 +5,4 @@
 --win-per-user-install
 --win-shortcut
 --win-shortcut-prompt
---win-update-url https://github.com/death-claw/vripper-project/releases
+--win-update-url https://github.com/dev-claw/vripper-project/releases

jpackage/jpackage.cfg

@@ -5,4 +5,4 @@
 --name VRipper
 --description "Image ripper tool for vipergirls"
 --dest dist
---vendor "death-claw"
+--vendor "dev-claw"

pom.xml

@@ -15,27 +15,29 @@
         <maven.compiler.source>${java.version}</maven.compiler.source>
         <maven.compiler.target>${java.version}</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <revision>5.8.0</revision>
-        <kotlin.version>1.9.22</kotlin.version>
-        <slf4j.version>2.0.12</slf4j.version>
-        <logback.version>1.5.0</logback.version>
-        <kotlinx-coroutines.version>1.8.0</kotlinx-coroutines.version>
-        <exposed.version>0.47.0</exposed.version>
+        <revision>6.5.3</revision>
+        <kotlin.version>2.0.21</kotlin.version>
+        <slf4j.version>2.0.16</slf4j.version>
+        <logback.version>1.5.12</logback.version>
+        <kotlinx-coroutines.version>1.9.0</kotlinx-coroutines.version>
+        <exposed.version>0.55.0</exposed.version>
         <snakeyaml.version>2.0</snakeyaml.version>
-        <koin.version>3.5.3</koin.version>
+        <koin.version>4.0.0</koin.version>
         <h2.version>2.2.224</h2.version>
         <htmlcleaner.version>2.29</htmlcleaner.version>
-        <failsafe.version>2.4.4</failsafe.version>
+        <failsafe.version>3.3.2</failsafe.version>
         <caffeine.version>3.1.8</caffeine.version>
         <jna-platform.version>5.14.0</jna-platform.version>
-        <kotlinx-serialization-json.version>1.6.2</kotlinx-serialization-json.version>
-        <httpclient5.version>5.3.1</httpclient5.version>
-        <commons-codec.version>1.16.1</commons-codec.version>
-        <liquibase-core.version>4.26.0</liquibase-core.version>
+        <kotlinx-serialization-json.version>1.7.3</kotlinx-serialization-json.version>
+        <httpclient5.version>5.4</httpclient5.version>
+        <httpcore5.version>5.3</httpcore5.version>
+        <liquibase-core.version>4.29.2</liquibase-core.version>
         <liquibase-slf4j.version>5.0.0</liquibase-slf4j.version>
         <grpc.kotlin.version>1.4.1</grpc.kotlin.version>
-        <java.grpc.version>1.62.2</java.grpc.version>
-        <protobuf.version>3.25.3</protobuf.version>
+        <java.grpc.version>1.68.1</java.grpc.version>
+        <protobuf.version>3.25.5</protobuf.version>
+        <sqlite-jdbc.version>3.47.0.0</sqlite-jdbc.version>
+        <bucket4j_jdk17-core.version>8.14.0</bucket4j_jdk17-core.version>
     </properties>
     <build>
         <plugins>
@@ -129,14 +131,19 @@
                 <groupId>com.h2database</groupId>
                 <version>${h2.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.xerial</groupId>
+                <artifactId>sqlite-jdbc</artifactId>
+                <version>${sqlite-jdbc.version}</version>
+            </dependency>
             <dependency>
                 <artifactId>htmlcleaner</artifactId>
                 <groupId>net.sourceforge.htmlcleaner</groupId>
                 <version>${htmlcleaner.version}</version>
             </dependency>
             <dependency>
+                <groupId>dev.failsafe</groupId>
                 <artifactId>failsafe</artifactId>
-                <groupId>net.jodah</groupId>
                 <version>${failsafe.version}</version>
             </dependency>
             <dependency>
@@ -170,9 +177,14 @@
                 <version>${httpclient5.version}</version>
             </dependency>
             <dependency>
-                <artifactId>commons-codec</artifactId>
-                <groupId>commons-codec</groupId>
-                <version>${commons-codec.version}</version>
+                <artifactId>httpcore5</artifactId>
+                <groupId>org.apache.httpcomponents.core5</groupId>
+                <version>${httpcore5.version}</version>
+            </dependency>
+            <dependency>
+                <artifactId>httpcore5-h2</artifactId>
+                <groupId>org.apache.httpcomponents.core5</groupId>
+                <version>${httpcore5.version}</version>
             </dependency>
             <dependency>
                 <artifactId>liquibase-core</artifactId>
@@ -194,15 +206,20 @@
                 <artifactId>grpc-protobuf</artifactId>
                 <version>${java.grpc.version}</version>
             </dependency>
+            <dependency>
+                <groupId>io.grpc</groupId>
+                <artifactId>grpc-netty-shaded</artifactId>
+                <version>${java.grpc.version}</version>
+            </dependency>
             <dependency>
                 <groupId>com.google.protobuf</groupId>
                 <artifactId>protobuf-kotlin</artifactId>
                 <version>${protobuf.version}</version>
             </dependency>
             <dependency>
-                <groupId>io.grpc</groupId>
-                <artifactId>grpc-netty-shaded</artifactId>
-                <version>${java.grpc.version}</version>
+                <groupId>com.bucket4j</groupId>
+                <artifactId>bucket4j_jdk17-core</artifactId>
+                <version>${bucket4j_jdk17-core.version}</version>
             </dependency>
         </dependencies>
     </dependencyManagement>

vripper-core/pom.xml

@@ -51,6 +51,10 @@
             <artifactId>h2</artifactId>
             <groupId>com.h2database</groupId>
         </dependency>
+        <dependency>
+            <groupId>org.xerial</groupId>
+            <artifactId>sqlite-jdbc</artifactId>
+        </dependency>
         <dependency>
             <artifactId>httpclient5</artifactId>
             <groupId>org.apache.httpcomponents.client5</groupId>
@@ -65,16 +69,12 @@
         </dependency>
         <dependency>
             <artifactId>failsafe</artifactId>
-            <groupId>net.jodah</groupId>
+            <groupId>dev.failsafe</groupId>
         </dependency>
         <dependency>
             <artifactId>caffeine</artifactId>
             <groupId>com.github.ben-manes.caffeine</groupId>
         </dependency>
-        <dependency>
-            <artifactId>commons-codec</artifactId>
-            <groupId>commons-codec</groupId>
-        </dependency>
         <dependency>
             <groupId>net.java.dev.jna</groupId>
             <artifactId>jna-platform</artifactId>
@@ -104,6 +104,10 @@
             <groupId>com.google.protobuf</groupId>
             <artifactId>protobuf-kotlin</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.bucket4j</groupId>
+            <artifactId>bucket4j_jdk17-core</artifactId>
+        </dependency>
     </dependencies>
 
     <build>
@@ -132,14 +136,14 @@
                 <executions>
                     <execution>
                         <id>compile</id>
-                        <phase>compile</phase>
+                        <phase>process-sources</phase>
                         <goals>
                             <goal>compile</goal>
                         </goals>
                     </execution>
                 </executions>
                 <configuration>
-                    <jvmTarget>1.8</jvmTarget>
+                    <jvmTarget>21</jvmTarget>
                     <compilerPlugins>
                         <plugin>kotlinx-serialization</plugin>
                     </compilerPlugins>

vripper-core/src/main/kotlin/me/vripper/Module.kt

@@ -1,10 +1,15 @@
 package me.vripper
 
-import me.vripper.download.DownloadService
+import me.vripper.data.repositories.ImageRepository
+import me.vripper.data.repositories.MetadataRepository
+import me.vripper.data.repositories.PostRepository
+import me.vripper.data.repositories.ThreadRepository
+import me.vripper.data.repositories.impl.ImageRepositoryImpl
+import me.vripper.data.repositories.impl.MetadataRepositoryImpl
+import me.vripper.data.repositories.impl.PostRepositoryImpl
+import me.vripper.data.repositories.impl.ThreadRepositoryImpl
 import me.vripper.event.EventBus
 import me.vripper.host.*
-import me.vripper.repositories.*
-import me.vripper.repositories.impl.*
 import me.vripper.services.*
 import org.koin.core.qualifier.named
 import org.koin.dsl.bind
@@ -17,11 +22,14 @@ val coreModule = module {
     single<SettingsService> {
         SettingsService(get())
     }
+    single<LogService> {
+        LogService(get())
+    }
     single<ImageRepository> {
         ImageRepositoryImpl()
     }
-    single<PostDownloadStateRepository> {
-        PostDownloadStateRepositoryImpl()
+    single<PostRepository> {
+        PostRepositoryImpl()
     }
     single<MetadataRepository> {
         MetadataRepositoryImpl()
@@ -29,41 +37,38 @@ val coreModule = module {
     single<ThreadRepository> {
         ThreadRepositoryImpl()
     }
-    single<LogRepository> {
-        LogRepositoryImpl(get())
-    }
     single<DataTransaction> {
-        DataTransaction(get(), get(), get(), get(), get(), get(), get())
+        DataTransaction(get(), get(), get(), get(), get(), get())
     }
     single<RetryPolicyService> {
-        RetryPolicyService(get(), get())
+        RetryPolicyService()
     }
     single<HTTPService> {
-        HTTPService(get(), get())
+        HTTPService()
     }
     single<VGAuthService> {
-        VGAuthService(get(), get(), get())
+        VGAuthService(get(), get())
     }
     single<ThreadCacheService> {
         ThreadCacheService(get())
     }
     single<DownloadService> {
-        DownloadService(get(), get(), get(), get(), get())
+        DownloadService(get(), get(), get(), get())
     }
     single<DownloadSpeedService> {
         DownloadSpeedService(get())
     }
 
     single<AppEndpointService> {
-        AppEndpointService(get(), get(), get(), get(), get())
+        AppEndpointService(get(), get(), get(), get(), get(), get())
     }
 
     single((named("localAppEndpointService"))) {
-        AppEndpointService(get(), get(), get(), get(), get())
+        AppEndpointService(get(), get(), get(), get(), get(), get())
     } bind IAppEndpointService::class
 
     single<MetadataService> {
-        MetadataService(get(), get(), get(), get())
+        MetadataService(get(), get())
     }
     single {
         AcidimgHost(get(), get(), get())

vripper-core/src/main/kotlin/me/vripper/RingAppender.kt

@@ -0,0 +1,16 @@
+package me.vripper
+
+import ch.qos.logback.classic.spi.ILoggingEvent
+import ch.qos.logback.core.AppenderBase
+import org.apache.commons.collections4.queue.CircularFifoQueue
+
+class RingAppender : AppenderBase<ILoggingEvent>() {
+
+    object Static {
+        val events = CircularFifoQueue<ILoggingEvent>()
+    }
+
+    override fun append(event: ILoggingEvent) {
+        Static.events.offer(event)
+    }
+}

vripper-core/src/main/kotlin/me/vripper/data/repositories/ImageRepository.kt

@@ -1,9 +1,9 @@
-package me.vripper.repositories
+package me.vripper.data.repositories
 
 import me.vripper.entities.ImageEntity
 import java.util.*
 
-interface ImageRepository {
+internal interface ImageRepository {
     fun save(imageEntity: ImageEntity): ImageEntity
     fun save(imageEntityList: List<ImageEntity>)
     fun deleteAllByPostId(postId: Long)

vripper-core/src/main/kotlin/me/vripper/data/repositories/MetadataRepository.kt

@@ -1,9 +1,9 @@
-package me.vripper.repositories
+package me.vripper.data.repositories
 
 import me.vripper.entities.MetadataEntity
 import java.util.*
 
-interface MetadataRepository {
+internal interface MetadataRepository {
     fun save(metadataEntity: MetadataEntity): MetadataEntity
     fun findByPostId(postId: Long): Optional<MetadataEntity>
     fun deleteByPostId(postId: Long): Int

vripper-core/src/main/kotlin/me/vripper/data/repositories/PostRepository.kt

@@ -1,12 +1,11 @@
-package me.vripper.repositories
+package me.vripper.data.repositories
 
 import me.vripper.entities.PostEntity
-import java.util.*
 
-interface PostDownloadStateRepository {
+internal interface PostRepository {
     fun save(postEntities: List<PostEntity>): List<PostEntity>
-    fun findByPostId(postId: Long): Optional<PostEntity>
-    fun findById(id: Long): Optional<PostEntity>
+    fun findByPostId(postId: Long): PostEntity?
+    fun findById(id: Long): PostEntity?
     fun findCompleted(): List<Long>
     fun findAll(): List<PostEntity>
     fun existByPostId(postId: Long): Boolean

vripper-core/src/main/kotlin/me/vripper/data/repositories/ThreadRepository.kt

@@ -1,9 +1,9 @@
-package me.vripper.repositories
+package me.vripper.data.repositories
 
 import me.vripper.entities.ThreadEntity
 import java.util.*
 
-interface ThreadRepository {
+internal interface ThreadRepository {
     fun save(threadEntity: ThreadEntity): ThreadEntity
     fun update(threadEntity: ThreadEntity)
     fun findByThreadId(threadId: Long): Optional<ThreadEntity>

vripper-core/src/main/kotlin/me/vripper/data/repositories/impl/ImageRepositoryImpl.kt

@@ -1,16 +1,16 @@
-package me.vripper.repositories.impl
+package me.vripper.data.repositories.impl
 
+import me.vripper.data.repositories.ImageRepository
+import me.vripper.data.tables.ImageTable
 import me.vripper.entities.ImageEntity
-import me.vripper.entities.domain.Status
-import me.vripper.repositories.ImageRepository
-import me.vripper.tables.ImageTable
+import me.vripper.entities.Status
 import org.jetbrains.exposed.sql.*
 import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
 import org.jetbrains.exposed.sql.transactions.TransactionManager
 import java.sql.Connection
 import java.util.*
 
-class ImageRepositoryImpl : ImageRepository {
+internal class ImageRepositoryImpl : ImageRepository {
     override fun save(imageEntity: ImageEntity): ImageEntity {
         val id = ImageTable.insertAndGetId {
             it[downloaded] = imageEntity.downloaded
@@ -47,21 +47,21 @@ class ImageRepositoryImpl : ImageRepository {
     }
 
     override fun findByPostId(postId: Long): List<ImageEntity> {
-        return ImageTable.select {
+        return ImageTable.selectAll().where {
             ImageTable.postId eq postId
         }.map(this::transform)
     }
 
     override fun countError(): Int {
         return ImageTable
-            .slice(ImageTable.id)
-            .select { ImageTable.status eq Status.ERROR.name }
+            .select(ImageTable.id)
+            .where { ImageTable.status eq Status.ERROR.name }
             .count().toInt()
     }
 
     override fun findByPostIdAndIsNotCompleted(postId: Long): List<ImageEntity> {
         return ImageTable
-            .select {
+            .selectAll().where {
                 (ImageTable.postId eq postId) and (ImageTable.status neq Status.FINISHED.name)
             }.map(this::transform)
     }
@@ -79,13 +79,13 @@ class ImageRepositoryImpl : ImageRepository {
     }
 
     override fun findByPostIdAndIsError(postId: Long): List<ImageEntity> {
-        return ImageTable.select {
+        return ImageTable.selectAll().where {
             (ImageTable.postId eq postId) and (ImageTable.status eq Status.ERROR.name)
         }.map(this::transform)
     }
 
     override fun findById(id: Long): Optional<ImageEntity> {
-        val result = ImageTable.select {
+        val result = ImageTable.selectAll().where {
             ImageTable.id eq id
         }.map(this::transform)
         return if (result.isEmpty()) {
@@ -122,7 +122,7 @@ class ImageRepositoryImpl : ImageRepository {
 
     override fun deleteAllByPostId(postIds: List<Long>) {
         val conn = TransactionManager.current().connection.connection as Connection
-        conn.prepareStatement("CREATE LOCAL TEMPORARY TABLE IMAGES_DELETE(POST_ID BIGINT PRIMARY KEY)")
+        conn.prepareStatement("CREATE TEMPORARY TABLE IMAGES_DELETE(POST_ID BIGINT PRIMARY KEY)")
             .use {
                 it.execute()
             }
@@ -140,7 +140,7 @@ class ImageRepositoryImpl : ImageRepository {
                 it.execute()
             }
 
-        conn.prepareStatement("TRUNCATE TABLE IMAGES_DELETE").use {
+        conn.prepareStatement("DELETE FROM IMAGES_DELETE").use {
             it.execute()
         }
     }

vripper-core/src/main/kotlin/me/vripper/data/repositories/impl/MetadataRepositoryImpl.kt

@@ -1,20 +1,20 @@
-package me.vripper.repositories.impl
+package me.vripper.data.repositories.impl
 
 import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
+import me.vripper.data.repositories.MetadataRepository
+import me.vripper.data.tables.MetadataTable
 import me.vripper.entities.MetadataEntity
-import me.vripper.repositories.MetadataRepository
-import me.vripper.tables.MetadataTable
 import org.jetbrains.exposed.sql.ResultRow
 import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
 import org.jetbrains.exposed.sql.deleteWhere
 import org.jetbrains.exposed.sql.insert
-import org.jetbrains.exposed.sql.select
+import org.jetbrains.exposed.sql.selectAll
 import org.jetbrains.exposed.sql.transactions.TransactionManager
 import java.sql.Connection
 import java.util.*
 
-class MetadataRepositoryImpl: MetadataRepository {
+internal class MetadataRepositoryImpl : MetadataRepository {
 
     override fun save(metadataEntity: MetadataEntity): MetadataEntity {
         MetadataTable.insert {
@@ -27,7 +27,7 @@ class MetadataRepositoryImpl: MetadataRepository {
 
     override fun findByPostId(postId: Long): Optional<MetadataEntity> {
 
-        val result = MetadataTable.select {
+        val result = MetadataTable.selectAll().where {
             MetadataTable.postId eq postId
         }.map(::transform)
 
@@ -50,7 +50,7 @@ class MetadataRepositoryImpl: MetadataRepository {
 
     override fun deleteAllByPostId(postIds: List<Long>) {
         val conn = TransactionManager.current().connection.connection as Connection
-        conn.prepareStatement("CREATE LOCAL TEMPORARY TABLE METADATA_DELETE(POST_ID BIGINT PRIMARY KEY)")
+        conn.prepareStatement("CREATE TEMPORARY TABLE METADATA_DELETE(POST_ID BIGINT PRIMARY KEY)")
             .use {
                 it.execute()
             }
@@ -68,7 +68,7 @@ class MetadataRepositoryImpl: MetadataRepository {
                 it.execute()
             }
 
-        conn.prepareStatement("TRUNCATE TABLE METADATA_DELETE").use {
+        conn.prepareStatement("DELETE FROM METADATA_DELETE").use {
             it.execute()
         }
     }

vripper-core/src/main/kotlin/me/vripper/data/repositories/impl/PostRepositoryImpl.kt

@@ -1,17 +1,16 @@
-package me.vripper.repositories.impl
+package me.vripper.data.repositories.impl
 
+import me.vripper.data.repositories.PostRepository
+import me.vripper.data.tables.PostTable
 import me.vripper.entities.PostEntity
-import me.vripper.entities.domain.Status
-import me.vripper.repositories.PostDownloadStateRepository
-import me.vripper.tables.PostTable
+import me.vripper.entities.Status
 import org.jetbrains.exposed.sql.*
 import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
 import org.jetbrains.exposed.sql.transactions.TransactionManager
 import java.sql.Connection
-import java.util.*
 
-class PostDownloadStateRepositoryImpl :
-    PostDownloadStateRepository {
+internal class PostRepositoryImpl :
+    PostRepository {
 
     private val delimiter = ";"
 
@@ -37,33 +36,24 @@ class PostDownloadStateRepositoryImpl :
         }.map(::transform)
     }
 
-    override fun findByPostId(postId: Long): Optional<PostEntity> {
-        val result = PostTable.select {
+    override fun findByPostId(postId: Long): PostEntity? {
+        val result = PostTable.selectAll().where {
             PostTable.postId eq postId
         }.map(::transform)
-        return if (result.isEmpty()) {
-            Optional.empty()
-        } else {
-            Optional.of(result.first())
-        }
+        return result.firstOrNull()
     }
 
     override fun findCompleted(): List<Long> {
-        return PostTable.slice(PostTable.postId).select {
+        return PostTable.select(PostTable.postId).where {
             (PostTable.status eq Status.FINISHED.name) and (PostTable.done greaterEq PostTable.total)
         }.map { it[PostTable.postId] }
     }
 
-    override fun findById(id: Long): Optional<PostEntity> {
-        val result = PostTable.select {
+    override fun findById(id: Long): PostEntity? {
+        val result = PostTable.selectAll().where {
             PostTable.id eq id
         }.map { transform(it) }
-
-        return if (result.isEmpty()) {
-            Optional.empty()
-        } else {
-            Optional.of(result.first())
-        }
+        return result.firstOrNull()
     }
 
     override fun findAll(): List<PostEntity> {
@@ -71,7 +61,7 @@ class PostDownloadStateRepositoryImpl :
     }
 
     override fun existByPostId(postId: Long): Boolean {
-        return PostTable.slice(PostTable.id).select { PostTable.postId eq postId }.count() > 0
+        return PostTable.select(PostTable.id).where { PostTable.postId eq postId }.count() > 0
     }
 
     override fun setDownloadingToStopped(): Int {
@@ -118,15 +108,14 @@ class PostDownloadStateRepositoryImpl :
     }
 
     override fun findMaxRank(): Int? {
-        return PostTable.slice(PostTable.rank.max())
-            .selectAll()
+        return PostTable.select(PostTable.rank.max())
             .firstOrNull()
             ?.get(PostTable.rank.max())
     }
 
     override fun deleteAll(postIds: List<Long>) {
         val conn = TransactionManager.current().connection.connection as Connection
-        conn.prepareStatement("CREATE LOCAL TEMPORARY TABLE POSTS_DELETE(POST_ID BIGINT PRIMARY KEY)")
+        conn.prepareStatement("CREATE TEMPORARY TABLE POSTS_DELETE(POST_ID BIGINT PRIMARY KEY)")
             .use {
                 it.execute()
             }
@@ -144,7 +133,7 @@ class PostDownloadStateRepositoryImpl :
                 it.execute()
             }
 
-        conn.prepareStatement("TRUNCATE TABLE POSTS_DELETE").use {
+        conn.prepareStatement("DELETE FROM POSTS_DELETE").use {
             it.execute()
         }
     }
@@ -156,7 +145,7 @@ class PostDownloadStateRepositoryImpl :
     }
 
     override fun findAllNonCompletedPostIds(): List<Long> {
-        return PostTable.slice(PostTable.postId).select {
+        return PostTable.select(PostTable.postId).where {
             PostTable.status neq Status.FINISHED.name
         }.map { it[PostTable.postId] }
     }

vripper-core/src/main/kotlin/me/vripper/data/repositories/impl/ThreadRepositoryImpl.kt

@@ -1,13 +1,13 @@
-package me.vripper.repositories.impl
+package me.vripper.data.repositories.impl
 
+import me.vripper.data.repositories.ThreadRepository
+import me.vripper.data.tables.ThreadTable
 import me.vripper.entities.ThreadEntity
-import me.vripper.repositories.ThreadRepository
-import me.vripper.tables.ThreadTable
 import org.jetbrains.exposed.sql.*
 import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
 import java.util.*
 
-class ThreadRepositoryImpl : ThreadRepository {
+internal class ThreadRepositoryImpl : ThreadRepository {
 
     override fun save(threadEntity: ThreadEntity): ThreadEntity {
         val id = ThreadTable.insertAndGetId {
@@ -27,7 +27,7 @@ class ThreadRepositoryImpl : ThreadRepository {
     }
 
     override fun findByThreadId(threadId: Long): Optional<ThreadEntity> {
-        val result = ThreadTable.select {
+        val result = ThreadTable.selectAll().where {
             ThreadTable.threadId eq threadId
         }.map(this::transform)
         return if (result.isEmpty()) {
@@ -42,7 +42,7 @@ class ThreadRepositoryImpl : ThreadRepository {
     }
 
     override fun findById(id: Long): Optional<ThreadEntity> {
-        val result = ThreadTable.select {
+        val result = ThreadTable.selectAll().where {
             ThreadTable.id eq id
         }.map(this::transform)
         return if (result.isEmpty()) {

vripper-core/src/main/kotlin/me/vripper/data/tables/ImageTable.kt

@@ -1,8 +1,8 @@
-package me.vripper.tables
+package me.vripper.data.tables
 
 import org.jetbrains.exposed.dao.id.LongIdTable
 
-object ImageTable : LongIdTable(name = "IMAGE", columnName = "ID") {
+internal object ImageTable : LongIdTable(name = "IMAGE", columnName = "ID") {
     val downloaded = long("DOWNLOADED")
     val host = byte("HOST")
     val index = integer("INDEX")
@@ -12,5 +12,5 @@ object ImageTable : LongIdTable(name = "IMAGE", columnName = "ID") {
     val size = long("SIZE")
     val url = varchar("URL", 200)
     val thumbUrl = varchar("THUMB_URL", 200)
-    val postIdRef = long("POST_ID_REF").references(PostTable.id, fkName = "IMAGE_POST_ID_REF_POST_ID_FK")
+    val postIdRef = long("POST_ID_REF").references(id, fkName = "IMAGE_POST_ID_REF_POST_ID_FK")
 }

vripper-core/src/main/kotlin/me/vripper/data/tables/MetadataTable.kt

@@ -1,8 +1,8 @@
-package me.vripper.tables
+package me.vripper.data.tables
 
 import org.jetbrains.exposed.sql.Table
 
-object MetadataTable : Table(name = "METADATA") {
+internal object MetadataTable : Table(name = "METADATA") {
     val postId = long("POST_ID")
     val data = varchar("DATA", 1_000_000)
 }

vripper-core/src/main/kotlin/me/vripper/data/tables/PostTable.kt

@@ -1,10 +1,10 @@
-package me.vripper.tables
+package me.vripper.data.tables
 
 import org.jetbrains.exposed.dao.id.LongIdTable
 import org.jetbrains.exposed.sql.javatime.datetime
 import java.time.LocalDateTime
 
-object PostTable : LongIdTable(name = "POST", columnName = "ID") {
+internal object PostTable : LongIdTable(name = "POST", columnName = "ID") {
     val done = integer("DONE")
     val hosts = varchar("HOSTS", 255)
     val outputPath = varchar("OUTPUT_PATH", 260)

vripper-core/src/main/kotlin/me/vripper/data/tables/ThreadTable.kt

@@ -1,8 +1,8 @@
-package me.vripper.tables
+package me.vripper.data.tables
 
 import org.jetbrains.exposed.dao.id.LongIdTable
 
-object ThreadTable : LongIdTable("THREAD", columnName = "ID") {
+internal object ThreadTable : LongIdTable("THREAD", columnName = "ID") {
     val total = integer("TOTAL").default(0)
     val url = varchar("URL", 200)
     val threadId = long("THREAD_ID")

vripper-core/src/main/kotlin/me/vripper/download/DownloadService.kt

@@ -1,299 +0,0 @@
-package me.vripper.download
-
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Runnable
-import kotlinx.coroutines.SupervisorJob
-import kotlinx.coroutines.launch
-import me.vripper.entities.ImageEntity
-import me.vripper.entities.LogEntryEntity
-import me.vripper.entities.PostEntity
-import me.vripper.entities.domain.Status
-import me.vripper.event.ErrorCountEvent
-import me.vripper.event.EventBus
-import me.vripper.event.QueueStateEvent
-import me.vripper.event.StoppedEvent
-import me.vripper.host.Host
-import me.vripper.model.ErrorCount
-import me.vripper.model.QueueState
-import me.vripper.services.DataTransaction
-import me.vripper.services.RetryPolicyService
-import me.vripper.services.SettingsService
-import me.vripper.services.VGAuthService
-import me.vripper.utilities.GLOBAL_EXECUTOR
-import me.vripper.utilities.formatToString
-import net.jodah.failsafe.Failsafe
-import net.jodah.failsafe.RetryPolicy
-import org.jetbrains.exposed.sql.transactions.transaction
-import java.util.concurrent.CompletableFuture
-import java.util.concurrent.locks.ReentrantLock
-import kotlin.concurrent.withLock
-
-class DownloadService(
-    private val settingsService: SettingsService,
-    private val dataTransaction: DataTransaction,
-    private val retryPolicyService: RetryPolicyService,
-    private val vgAuthService: VGAuthService,
-    private val eventBus: EventBus
-) {
-    private val maxPoolSize: Int = 24
-    private val log by me.vripper.delegate.LoggerDelegate()
-    private val running: MutableMap<Byte, MutableList<ImageDownloadRunnable>> = mutableMapOf()
-    private val pending: MutableMap<Byte, MutableList<ImageDownloadRunnable>> = mutableMapOf()
-    private val lock = ReentrantLock()
-    private val condition = lock.newCondition()
-    private val coroutineScope = CoroutineScope(SupervisorJob())
-
-    init {
-        Thread.ofVirtual().name("Download Loop").unstarted(Runnable {
-            val accepted: MutableList<ImageDownloadRunnable> = mutableListOf()
-            val candidates: MutableList<ImageDownloadRunnable> = mutableListOf()
-            while (!Thread.currentThread().isInterrupted) {
-                lock.withLock {
-                    candidates.addAll(getCandidates(candidateCount()))
-                    candidates.forEach {
-                        if (canRun(it.context.imageEntity.host)) {
-                            accepted.add(it)
-                            running[it.context.imageEntity.host]!!.add(it)
-                            log.debug("${it.context.imageEntity.url} accepted to run")
-                        }
-                    }
-                    accepted.forEach {
-                        pending[it.context.imageEntity.host]?.remove(it)
-                        scheduleForDownload(it)
-                    }
-                    accepted.clear()
-                    candidates.clear()
-                    try {
-                        condition.await()
-                    } catch (e: InterruptedException) {
-                        Thread.currentThread().interrupt()
-                    }
-                }
-            }
-        }).start()
-    }
-
-    fun stop(postIds: List<Long> = emptyList()) {
-        if (postIds.isNotEmpty()) {
-            stopInternal(postIds)
-            coroutineScope.launch {
-                eventBus.publishEvent(StoppedEvent(postIds))
-            }
-        } else {
-            stopAll()
-            coroutineScope.launch {
-                eventBus.publishEvent(StoppedEvent(listOf(-1)))
-            }
-        }
-    }
-
-    fun restartAll(postEntityIds: List<PostEntity> = emptyList()) {
-        if (postEntityIds.isNotEmpty()) {
-            restart(postEntityIds.associateWith { dataTransaction.findByPostIdAndIsNotCompleted(it.postId) })
-        } else {
-            restart(dataTransaction.findAllPosts()
-                .associateWith { dataTransaction.findByPostIdAndIsNotCompleted(it.postId) })
-        }
-    }
-
-    private fun restart(posts: Map<PostEntity, List<ImageEntity>>) {
-        lock.withLock {
-            val toProcess = mutableMapOf<PostEntity, List<ImageEntity>>()
-
-            for ((post, images) in posts) {
-                if (images.isEmpty()) {
-                    continue
-                }
-                if (isPending(post.postId)) {
-                    continue
-                }
-                toProcess[post] = images
-            }
-
-            toProcess.forEach { (post, images) ->
-                post.status = Status.PENDING
-                images.forEach { image ->
-                    with(image) {
-                        this.status = Status.PENDING
-                        this.downloaded = 0
-                    }
-                }
-            }
-
-            transaction {
-                dataTransaction.updatePosts(
-                    toProcess.keys.toList()
-                )
-                dataTransaction.updateImages(
-                    toProcess.values.flatten()
-                )
-            }
-
-            toProcess.keys.forEach {
-                vgAuthService.leaveThanks(it)
-            }
-
-
-            toProcess.entries.forEach { (post, images) ->
-                images.forEach { image ->
-                    log.debug("Enqueuing a job for ${image.url}")
-                    val imageDownloadRunnable = ImageDownloadRunnable(
-                        image, post.rank, settingsService.settings
-                    )
-                    pending.computeIfAbsent(
-                        image.host
-                    ) { mutableListOf() }
-                    pending[image.host]!!.add(imageDownloadRunnable)
-                }
-            }
-
-            condition.signal()
-        }
-    }
-
-    private fun isPending(postId: Long): Boolean {
-        return pending.values.flatten().any { it.context.imageEntity.postId == postId }
-    }
-
-    private fun isRunning(postId: Long): Boolean {
-        return running.values.flatten().any { it.context.imageEntity.postId == postId }
-    }
-
-    private fun stopAll() {
-        lock.withLock {
-            pending.values.clear()
-            running.values.flatten().forEach { obj: ImageDownloadRunnable -> obj.stop() }
-            dataTransaction.findAllNonCompletedPostIds().forEach {
-                dataTransaction.stopImagesByPostIdAndIsNotCompleted(it)
-                dataTransaction.finishPost(it)
-            }
-        }
-    }
-
-    private fun stopInternal(postIds: List<Long>) {
-        lock.withLock {
-            for (postId in postIds) {
-                pending.values.forEach { pending ->
-                    pending.removeIf { it.context.imageEntity.postId == postId }
-                }
-                running.values.flatten()
-                    .filter { p: ImageDownloadRunnable -> p.context.imageEntity.postId == postId }
-                    .forEach { obj: ImageDownloadRunnable -> obj.stop() }
-            }
-            postIds.forEach {
-                dataTransaction.stopImagesByPostIdAndIsNotCompleted(it)
-                dataTransaction.finishPost(it)
-            }
-        }
-    }
-
-    private fun canRun(host: Byte): Boolean {
-        val totalRunning = running.values.sumOf { it.size }
-        return (running[host]!!.size < settingsService.settings.connectionSettings.maxConcurrentPerHost && if (settingsService.settings.connectionSettings.maxGlobalConcurrent == 0) totalRunning < maxPoolSize else totalRunning < settingsService.settings.connectionSettings.maxGlobalConcurrent)
-    }
-
-    private fun candidateCount(): Map<Byte, Int> {
-        val map: MutableMap<Byte, Int> = mutableMapOf()
-        Host.getHosts().values.forEach { host: Byte ->
-            val imageDownloadRunnableList: List<ImageDownloadRunnable> = running.computeIfAbsent(
-                host
-            ) { mutableListOf() }
-            val count: Int =
-                settingsService.settings.connectionSettings.maxConcurrentPerHost - imageDownloadRunnableList.size
-            log.debug("Download slots for $host: $count")
-            map[host] = count
-        }
-        return map
-    }
-
-    private fun getCandidates(candidateCount: Map<Byte, Int>): List<ImageDownloadRunnable> {
-        val hostIntegerMap: MutableMap<Byte, Int> = candidateCount.toMutableMap()
-        val candidates: MutableList<ImageDownloadRunnable> = mutableListOf()
-        hosts@ for (host in pending.keys) {
-
-            val list: List<ImageDownloadRunnable> =
-                pending[host]!!.sortedWith(Comparator.comparingInt<ImageDownloadRunnable> { it.postRank }
-                    .thenComparingInt { it.context.imageEntity.index })
-
-            for (imageDownloadRunnable in list) {
-                val count = hostIntegerMap[host] ?: 0
-                if (count > 0) {
-                    candidates.add(imageDownloadRunnable)
-                    hostIntegerMap[host] = count - 1
-                } else {
-                    continue@hosts
-                }
-            }
-        }
-        return candidates
-    }
-
-    private fun scheduleForDownload(imageDownloadRunnable: ImageDownloadRunnable) {
-        log.debug("Scheduling a job for ${imageDownloadRunnable.context.imageEntity.url}")
-        CompletableFuture.runAsync({
-            coroutineScope.launch {
-                eventBus.publishEvent(QueueStateEvent(QueueState(runningCount(), pendingCount())))
-            }
-            Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicyForDownload())
-                .onFailure {
-                    try {
-                        dataTransaction.saveLog(
-                            LogEntryEntity(
-                                type = LogEntryEntity.Type.DOWNLOAD,
-                                status = LogEntryEntity.Status.ERROR,
-                                message = "Failed to download ${imageDownloadRunnable.context.imageEntity.url}\n ${it.failure.formatToString()}"
-                            )
-                        )
-                    } catch (exp: Exception) {
-                        log.error("Failed to save event", exp)
-                    }
-                    log.error(
-                        "Failed to download ${imageDownloadRunnable.context.imageEntity.url} after ${it.attemptCount} tries",
-                        it.failure
-                    )
-                    val image = imageDownloadRunnable.context.imageEntity
-                    image.status = Status.ERROR
-                    dataTransaction.updateImage(image)
-                }.onComplete {
-                    afterJobFinish(imageDownloadRunnable)
-                    coroutineScope.launch {
-                        eventBus.publishEvent(
-                            QueueStateEvent(
-                                QueueState(
-                                    runningCount(), pendingCount()
-                                )
-                            )
-                        )
-                    }
-                    coroutineScope.launch {
-                        eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataTransaction.countImagesInError())))
-                    }
-                    log.debug(
-                        "Finished downloading ${imageDownloadRunnable.context.imageEntity.url}"
-                    )
-                }.run(imageDownloadRunnable::run)
-        }, GLOBAL_EXECUTOR)
-    }
-
-    private fun afterJobFinish(imageDownloadRunnable: ImageDownloadRunnable) {
-        lock.withLock {
-            val image = imageDownloadRunnable.context.imageEntity
-            running[image.host]!!.remove(imageDownloadRunnable)
-            if (!isPending(image.postId) && !isRunning(
-                    image.postId
-                ) && !imageDownloadRunnable.context.stopped
-            ) {
-                dataTransaction.finishPost(image.postId, true)
-            }
-            condition.signal()
-        }
-    }
-
-    private fun pendingCount(): Int {
-        return pending.values.sumOf { it.size }
-    }
-
-    private fun runningCount(): Int {
-        return running.values.sumOf { it.size }
-    }
-}

vripper-core/src/main/kotlin/me/vripper/download/ImageDownloadContext.kt

@@ -1,17 +0,0 @@
-package me.vripper.download
-
-import me.vripper.entities.ImageEntity
-import me.vripper.model.Settings
-import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase
-import org.apache.hc.client5.http.cookie.BasicCookieStore
-import org.apache.hc.client5.http.protocol.HttpClientContext
-import org.koin.core.component.KoinComponent
-
-class ImageDownloadContext(val imageEntity: ImageEntity, val settings: Settings) : KoinComponent {
-
-    val httpContext: HttpClientContext =
-        HttpClientContext.create().apply { cookieStore = BasicCookieStore() }
-    val requests = mutableListOf<HttpUriRequestBase>()
-    val postId = imageEntity.postIdRef
-    var stopped = false
-}

vripper-core/src/main/kotlin/me/vripper/download/ImageDownloadRunnable.kt

@@ -1,149 +0,0 @@
-package me.vripper.download
-
-import kotlinx.coroutines.*
-import me.vripper.entities.ImageEntity
-import me.vripper.entities.domain.Status
-import me.vripper.exception.DownloadException
-import me.vripper.exception.HostException
-import me.vripper.host.DownloadedImage
-import me.vripper.host.Host
-import me.vripper.host.ImageMimeType
-import me.vripper.model.Settings
-import me.vripper.services.*
-import me.vripper.utilities.PathUtils.getExtension
-import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
-import me.vripper.utilities.PathUtils.sanitize
-import net.jodah.failsafe.function.CheckedRunnable
-import org.koin.core.component.KoinComponent
-import org.koin.core.component.inject
-import java.io.IOException
-import java.nio.file.Files
-import java.nio.file.Path
-import java.nio.file.StandardCopyOption
-import java.util.*
-import kotlin.io.path.Path
-import kotlin.io.path.pathString
-
-class ImageDownloadRunnable(
-    private val imageEntity: ImageEntity, val postRank: Int, private val settings: Settings
-) : KoinComponent, CheckedRunnable {
-    private val log by me.vripper.delegate.LoggerDelegate()
-
-    private val dataTransaction: DataTransaction by inject()
-    private val hosts: List<Host> = getKoin().getAll()
-
-    val context: ImageDownloadContext = ImageDownloadContext(imageEntity, settings)
-    private var stopped: Boolean
-        get() = context.stopped
-        set(value) {
-            context.stopped = value
-        }
-
-    @Throws(DownloadException::class)
-    fun download() {
-        try {
-            imageEntity.status = Status.DOWNLOADING
-            imageEntity.downloaded = 0
-            dataTransaction.updateImage(imageEntity)
-            synchronized(imageEntity.postId.toString().intern()) {
-                val post = dataTransaction.findPostById(context.postId).orElseThrow()
-                if (post.status != Status.DOWNLOADING) {
-                    post.status = Status.DOWNLOADING
-                    dataTransaction.updatePost(post)
-                }
-            }
-            log.debug("Getting image url and name from ${imageEntity.url} using ${imageEntity.host}")
-            val host = hosts.first { it.isSupported(imageEntity.url) }
-            val downloadedImage = host.downloadInternal(imageEntity.url, context)
-            log.debug("Resolved name for ${imageEntity.url}: ${downloadedImage.name}")
-            log.debug("Downloaded image {} to {}", imageEntity.url, downloadedImage.path)
-            synchronized(imageEntity.postId.toString().intern()) {
-                val post = dataTransaction.findPostById(context.postId).orElseThrow()
-                val downloadDirectory = Path(post.downloadDirectory, post.folderName).pathString
-                checkImageTypeAndRename(
-                    downloadDirectory, downloadedImage, imageEntity.index
-                )
-                if (imageEntity.downloaded == imageEntity.size && imageEntity.size > 0) {
-                    imageEntity.status = Status.FINISHED
-                    post.done += 1
-                    post.downloaded += imageEntity.size
-                    dataTransaction.updatePost(post)
-                } else {
-                    imageEntity.status = Status.ERROR
-                }
-                dataTransaction.updateImage(imageEntity)
-            }
-        } catch (e: Exception) {
-            if (stopped) {
-                return
-            }
-            imageEntity.status = Status.ERROR
-            dataTransaction.updateImage(imageEntity)
-            throw DownloadException(e)
-        }
-    }
-
-    @Throws(HostException::class)
-    private fun checkImageTypeAndRename(
-        downloadDirectory: String, downloadedImage: DownloadedImage, index: Int
-    ) {
-        val existingExtension = getExtension(downloadedImage.name).lowercase()
-        val fileNameWithoutExtension = getFileNameWithoutExtension(downloadedImage.name)
-        val extension = when (downloadedImage.type) {
-            ImageMimeType.IMAGE_BMP -> "BMP"
-            ImageMimeType.IMAGE_GIF -> "GIF"
-            ImageMimeType.IMAGE_JPEG -> "JPG"
-            ImageMimeType.IMAGE_PNG -> "PNG"
-            ImageMimeType.IMAGE_WEBP -> "WEBP"
-        }
-        val filename =
-            if (existingExtension.isBlank()) "${sanitize(downloadedImage.name)}.$extension" else "${
-                sanitize(
-                    fileNameWithoutExtension
-                )
-            }.$extension"
-        try {
-            val downloadDestinationFolder = Path.of(downloadDirectory)
-            Files.createDirectories(downloadDestinationFolder)
-            val finalFilename = "${
-                if (settings.downloadSettings.forceOrder) String.format(
-                    "%03d_", index + 1
-                ) else ""
-            }$filename"
-            imageEntity.filename = finalFilename
-            val imageDownloadPath = downloadDestinationFolder.resolve(finalFilename)
-            Files.copy(downloadedImage.path, imageDownloadPath, StandardCopyOption.REPLACE_EXISTING)
-        } catch (e: Exception) {
-            throw HostException("Failed to rename the image", e)
-        } finally {
-            try {
-                Files.delete(downloadedImage.path)
-            } catch (_: IOException) {
-            }
-        }
-    }
-
-    @Throws(Exception::class)
-    override fun run() {
-        if (stopped) {
-            return
-        }
-        download()
-    }
-
-    override fun equals(other: Any?): Boolean {
-        if (this === other) return true
-        if (other == null || javaClass != other.javaClass) return false
-        val that = other as ImageDownloadRunnable
-        return imageEntity.id == that.imageEntity.id
-    }
-
-    override fun hashCode(): Int {
-        return Objects.hash(imageEntity.id)
-    }
-
-    fun stop() {
-        context.requests.forEach { it.abort() }
-        stopped = true
-    }
-}

vripper-core/src/main/kotlin/me/vripper/entities/ImageEntity.kt

@@ -1,7 +1,8 @@
 package me.vripper.entities
 
-import me.vripper.entities.domain.Status
+import kotlinx.serialization.Serializable
 
+@Serializable
 data class ImageEntity(
     val id: Long = -1,
     val postId: Long,

vripper-core/src/main/kotlin/me/vripper/entities/LogEntryEntity.kt

@@ -1,22 +1,14 @@
 package me.vripper.entities
 
-import java.time.LocalDateTime
+import kotlinx.serialization.Serializable
 
+@Serializable
 data class LogEntryEntity(
-    val id: Long = -1,
-    val type: Type,
-    val status: Status,
-    val time: LocalDateTime = LocalDateTime.now(),
-    val message: String,
-) {
-
-    enum class Type(val stringValue: String) {
-        POST("Post"), THREAD("Thread"), THANKS("Thanks"), METADATA("Metadata"), SCAN("Scan"), DOWNLOAD(
-            "Download"
-        )
-    }
-
-    enum class Status(val stringValue: String) {
-        PENDING("Pending"), PROCESSING("Processing"), DONE("Done"), ERROR("Error")
-    }
-}
+    val sequence: Long,
+    val timestamp: String,
+    val threadName: String,
+    val loggerName: String,
+    val levelString: String,
+    val formattedMessage: String,
+    val throwable: String,
+)

vripper-core/src/main/kotlin/me/vripper/entities/MetadataEntity.kt

@@ -2,6 +2,7 @@ package me.vripper.entities
 
 import kotlinx.serialization.Serializable
 
+@Serializable
 data class MetadataEntity(val postId: Long, val data: Data) {
     @Serializable
     data class Data(

vripper-core/src/main/kotlin/me/vripper/entities/PostEntity.kt

@@ -1,8 +1,10 @@
 package me.vripper.entities
 
-import me.vripper.entities.domain.Status
+import kotlinx.serialization.Contextual
+import kotlinx.serialization.Serializable
 import java.time.LocalDateTime
 
+@Serializable
 data class PostEntity(
     val id: Long = -1,
     val postTitle: String,
@@ -15,7 +17,7 @@ data class PostEntity(
     val total: Int,
     val hosts: Set<String>,
     val downloadDirectory: String,
-    val addedOn: LocalDateTime = LocalDateTime.now(),
+    @Contextual val addedOn: LocalDateTime = LocalDateTime.now(),
     var folderName: String,
     var status: Status = Status.STOPPED,
     var done: Int = 0,

vripper-core/src/main/kotlin/me/vripper/entities/Status.kt

@@ -1,4 +1,4 @@
-package me.vripper.entities.domain
+package me.vripper.entities
 
 enum class Status(val stringValue: String) {
     PENDING("Pending"), DOWNLOADING("Downloading"), FINISHED("Finished"), ERROR("Error"), STOPPED("Stopped")

vripper-core/src/main/kotlin/me/vripper/entities/ThreadEntity.kt

@@ -1,5 +1,8 @@
 package me.vripper.entities
 
+import kotlinx.serialization.Serializable
+
+@Serializable
 data class ThreadEntity(
     val id: Long = -1,
     val title: String,

vripper-core/src/main/kotlin/me/vripper/event/Event.kt

@@ -21,7 +21,5 @@ data class ErrorCountEvent(val errorCount: ErrorCount)
 data class SettingsUpdateEvent(val settings: Settings)
 data class MetadataUpdateEvent(val metadataEntity: MetadataEntity)
 data class LogCreateEvent(val logEntryEntity: LogEntryEntity)
-data class LogUpdateEvent(val logEntryEntity: LogEntryEntity)
-data class LogDeleteEvent(val deleted: List<Long>)
 data class LoadingTasks(val loading: Boolean)
 data class StoppedEvent(val postIds: List<Long>)

vripper-core/src/main/kotlin/me/vripper/event/EventBus.kt

@@ -1,14 +1,15 @@
 package me.vripper.event
 
+import kotlinx.coroutines.channels.BufferOverflow
 import kotlinx.coroutines.flow.MutableSharedFlow
 import kotlinx.coroutines.flow.asSharedFlow
 
 object EventBus {
 
-    private val _events = MutableSharedFlow<Any>()
+    private val _events = MutableSharedFlow<Any>(0, Int.MAX_VALUE, BufferOverflow.DROP_OLDEST)
     val events = _events.asSharedFlow()
 
-    suspend fun publishEvent(event: Any) {
-        _events.emit(event)
+    fun publishEvent(event: Any) {
+        _events.tryEmit(event)
     }
 }

vripper-core/src/main/kotlin/me/vripper/host/AcidimgHost.kt

@@ -1,10 +1,13 @@
 package me.vripper.host
 
-import me.vripper.download.ImageDownloadContext
 import me.vripper.exception.HostException
 import me.vripper.exception.XpathException
-import me.vripper.services.*
+import me.vripper.services.DataTransaction
+import me.vripper.services.DownloadService.ImageDownloadContext
+import me.vripper.services.DownloadSpeedService
+import me.vripper.services.HTTPService
 import me.vripper.utilities.HtmlUtils
+import me.vripper.utilities.LoggerDelegate
 import me.vripper.utilities.XpathUtils
 import org.apache.hc.client5.http.classic.methods.HttpPost
 import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
@@ -12,12 +15,12 @@ import org.apache.hc.core5.http.message.BasicNameValuePair
 import org.w3c.dom.Document
 import org.w3c.dom.Node
 
-class AcidimgHost(
+internal class AcidimgHost(
     private val httpService: HTTPService,
     dataTransaction: DataTransaction,
     downloadSpeedService: DownloadSpeedService,
 ) : Host("acidimg.cc", 0, httpService, dataTransaction, downloadSpeedService) {
-    private val log by me.vripper.delegate.LoggerDelegate()
+    private val log by LoggerDelegate()
 
     @Throws(HostException::class)
     override fun resolve(
@@ -45,7 +48,7 @@ class AcidimgHost(
                 )
             )
         }.also { context.requests.add(it) }
-        log.debug(String.format("Requesting %s", httpPost))
+        log.debug(String.format("Requesting %s", httpPost.uri))
         val doc = try {
             httpService.client.execute(
                 httpPost, context.httpContext

vripper-core/src/main/kotlin/me/vripper/host/DPicMeHost.kt

@@ -1,20 +1,23 @@
 package me.vripper.host
 
-import me.vripper.download.ImageDownloadContext
 import me.vripper.exception.HostException
 import me.vripper.exception.XpathException
-import me.vripper.services.*
+import me.vripper.services.DataTransaction
+import me.vripper.services.DownloadService.ImageDownloadContext
+import me.vripper.services.DownloadSpeedService
+import me.vripper.services.HTTPService
+import me.vripper.utilities.LoggerDelegate
 import me.vripper.utilities.XpathUtils
 import org.w3c.dom.Document
 import org.w3c.dom.Node
 import java.util.*
 
-class DPicMeHost(
+internal class DPicMeHost(
     httpService: HTTPService,
     dataTransaction: DataTransaction,
     downloadSpeedService: DownloadSpeedService,
 ) : Host("dpic.me", 1, httpService, dataTransaction, downloadSpeedService) {
-    private val log by me.vripper.delegate.LoggerDelegate()
+    private val log by LoggerDelegate()
 
     @Throws(HostException::class)
     override fun resolve(

vripper-core/src/main/kotlin/me/vripper/host/Host.kt

@@ -1,10 +1,17 @@
 package me.vripper.host
 
-import me.vripper.download.ImageDownloadContext
+import kotlinx.coroutines.cancelAndJoin
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.isActive
+import kotlinx.coroutines.runBlocking
 import me.vripper.exception.DownloadException
 import me.vripper.exception.HostException
-import me.vripper.services.*
+import me.vripper.services.DataTransaction
+import me.vripper.services.DownloadService.ImageDownloadContext
+import me.vripper.services.DownloadSpeedService
+import me.vripper.services.HTTPService
 import me.vripper.utilities.HtmlUtils
+import me.vripper.utilities.LoggerDelegate
 import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
 import org.apache.hc.client5.http.classic.methods.HttpGet
 import org.apache.hc.client5.http.classic.methods.HttpHead
@@ -12,17 +19,18 @@ import org.apache.hc.core5.http.ClassicHttpResponse
 import org.apache.hc.core5.http.Header
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.w3c.dom.Document
+import java.io.BufferedOutputStream
 import java.nio.file.Files
 import java.nio.file.Path
 
-abstract class Host(
+internal abstract class Host(
     val hostName: String,
     val hostId: Byte,
     private val httpService: HTTPService,
     private val dataTransaction: DataTransaction,
     private val downloadSpeedService: DownloadSpeedService
 ) {
-    private val log by me.vripper.delegate.LoggerDelegate()
+    private val log by LoggerDelegate()
 
     companion object {
         private const val READ_BUFFER_SIZE = 8192
@@ -94,10 +102,10 @@ abstract class Host(
             "vripper_",
             ".tmp"
         )
-        return Files.newOutputStream(tempImage).use { fos ->
+        return BufferedOutputStream(Files.newOutputStream(tempImage)).use { bos ->
             val image = context.imageEntity
             synchronized(image.postId.toString().intern()) {
-                val post = dataTransaction.findPostById(context.postId).orElseThrow()
+                val post = dataTransaction.findPostById(context.postId)
                 val size = if (image.size < 0) {
                     response.entity.contentLength
                 } else {
@@ -118,14 +126,23 @@ abstract class Host(
             )
             val buffer = ByteArray(READ_BUFFER_SIZE)
             var read: Int
-            while (response.entity.content.read(buffer, 0, READ_BUFFER_SIZE)
-                    .also { read = it } != -1 && !context.stopped
+            val reporterJob = context.launchCoroutine {
+                while (isActive) {
+                    dataTransaction.updateImage(image, false)
+                    delay(100)
+                }
+            }
+            while (response.entity.content.read(buffer)
+                    .also { read = it } != -1
             ) {
-                fos.write(buffer, 0, read)
+                bos.write(buffer, 0, read)
                 image.downloaded += read
-                dataTransaction.updateImage(image)
                 downloadSpeedService.reportDownloadedBytes(read.toLong())
             }
+            runBlocking {
+                reporterJob.cancelAndJoin()
+            }
+            dataTransaction.updateImage(image)
             Pair(tempImage, mimeType)
         }
     }

vripper-core/src/main/kotlin/me/vripper/host/ImageBamHost.kt

@@ -1,10 +1,13 @@
 package me.vripper.host
 
-import me.vripper.download.ImageDownloadContext
 import me.vripper.exception.HostException
 import me.vripper.exception.XpathException
-import me.vripper.services.*
+import me.vripper.services.DataTransaction
+import me.vripper.services.DownloadService.ImageDownloadContext
+import me.vripper.services.DownloadSpeedService
+import me.vripper.services.HTTPService
 import me.vripper.utilities.HtmlUtils
+import me.vripper.utilities.LoggerDelegate
 import me.vripper.utilities.XpathUtils
 import org.apache.hc.client5.http.impl.cookie.BasicClientCookie
 import org.w3c.dom.Document
@@ -14,12 +17,12 @@ import java.time.LocalDateTime
 import java.time.ZoneId
 import java.util.*
 
-class ImageBamHost(
+internal class ImageBamHost(
     httpService: HTTPService,
     dataTransaction: DataTransaction,
     downloadSpeedService: DownloadSpeedService,
 ) : Host("imagebam.com", 2, httpService, dataTransaction, downloadSpeedService) {
-    private val log by me.vripper.delegate.LoggerDelegate()
+    private val log by LoggerDelegate()
 
     @Throws(HostException::class)
     override fun resolve(

vripper-core/src/main/kotlin/me/vripper/host/ImageTwistHost.kt

@@ -1,20 +1,23 @@
 package me.vripper.host
 
-import me.vripper.download.ImageDownloadContext
 import me.vripper.exception.HostException
 import me.vripper.exception.XpathException
-import me.vripper.services.*
+import me.vripper.services.DataTransaction
+import me.vripper.services.DownloadService.ImageDownloadContext
+import me.vripper.services.DownloadSpeedService
+import me.vripper.services.HTTPService
+import me.vripper.utilities.LoggerDelegate
 import me.vripper.utilities.XpathUtils
 import org.w3c.dom.Document
 import org.w3c.dom.Node
 import java.util.*
 
-class ImageTwistHost(
+internal class ImageTwistHost(
     httpService: HTTPService,
     dataTransaction: DataTransaction,
     downloadSpeedService: DownloadSpeedService,
 ) : Host("imagetwist.com", 3, httpService, dataTransaction, downloadSpeedService) {
-    private val log by me.vripper.delegate.LoggerDelegate()
+    private val log by LoggerDelegate()
 
     @Throws(HostException::class)
     override fun resolve(

vripper-core/src/main/kotlin/me/vripper/host/ImageVenueHost.kt

@@ -1,20 +1,23 @@
 package me.vripper.host
 
-import me.vripper.download.ImageDownloadContext
 import me.vripper.exception.HostException
 import me.vripper.exception.XpathException
-import me.vripper.services.*
+import me.vripper.services.DataTransaction
+import me.vripper.services.DownloadService.ImageDownloadContext
+import me.vripper.services.DownloadSpeedService
+import me.vripper.services.HTTPService
 import me.vripper.utilities.HtmlUtils
+import me.vripper.utilities.LoggerDelegate
 import me.vripper.utilities.XpathUtils
 import org.w3c.dom.Document
 import org.w3c.dom.Node
 
-class ImageVenueHost(
+internal class ImageVenueHost(
     httpService: HTTPService,
     dataTransaction: DataTransaction,
     downloadSpeedService: DownloadSpeedService,
 ) : Host("imagevenue.com", 4, httpService, dataTransaction, downloadSpeedService) {
-    private val log by me.vripper.delegate.LoggerDelegate()
+    private val log by LoggerDelegate()
 
     @Throws(HostException::class)
     override fun resolve(

vripper-core/src/main/kotlin/me/vripper/host/ImageZillaHost.kt

@@ -1,18 +1,21 @@
 package me.vripper.host
 
-import me.vripper.download.ImageDownloadContext
 import me.vripper.exception.HostException
 import me.vripper.exception.XpathException
-import me.vripper.services.*
+import me.vripper.services.DataTransaction
+import me.vripper.services.DownloadService.ImageDownloadContext
+import me.vripper.services.DownloadSpeedService
+import me.vripper.services.HTTPService
+import me.vripper.utilities.LoggerDelegate
 import me.vripper.utilities.XpathUtils
 import org.w3c.dom.Document
 
-class ImageZillaHost(
+internal class ImageZillaHost(
     httpService: HTTPService,
     dataTransaction: DataTransaction,
     downloadSpeedService: DownloadSpeedService,
 ) : Host("imagezilla.net", 5, httpService, dataTransaction, downloadSpeedService) {
-    private val log by me.vripper.delegate.LoggerDelegate()
+    private val log by LoggerDelegate()
 
     @Throws(HostException::class)
     override fun resolve(

vripper-core/src/main/kotlin/me/vripper/host/ImgSpiceHost.kt

@@ -1,19 +1,22 @@
 package me.vripper.host
 
-import me.vripper.download.ImageDownloadContext
 import me.vripper.exception.HostException
 import me.vripper.exception.XpathException
-import me.vripper.services.*
+import me.vripper.services.DataTransaction
+import me.vripper.services.DownloadService.ImageDownloadContext
+import me.vripper.services.DownloadSpeedService
+import me.vripper.services.HTTPService
+import me.vripper.utilities.LoggerDelegate
 import me.vripper.utilities.XpathUtils
 import org.w3c.dom.Document
 import org.w3c.dom.Node
 
-class ImgSpiceHost(
+internal class ImgSpiceHost(
     httpService: HTTPService,
     dataTransaction: DataTransaction,
     downloadSpeedService: DownloadSpeedService,
 ) : Host("imgspice.com", 7, httpService, dataTransaction, downloadSpeedService) {
-    private val log by me.vripper.delegate.LoggerDelegate()
+    private val log by LoggerDelegate()
 
     @Throws(HostException::class)
     override fun resolve(

vripper-core/src/main/kotlin/me/vripper/host/ImgboxHost.kt

@@ -1,19 +1,22 @@
 package me.vripper.host
 
-import me.vripper.download.ImageDownloadContext
 import me.vripper.exception.HostException
 import me.vripper.exception.XpathException
-import me.vripper.services.*
+import me.vripper.services.DataTransaction
+import me.vripper.services.DownloadService.ImageDownloadContext
+import me.vripper.services.DownloadSpeedService
+import me.vripper.services.HTTPService
+import me.vripper.utilities.LoggerDelegate
 import me.vripper.utilities.XpathUtils
 import org.w3c.dom.Document
 import org.w3c.dom.Node
 
-class ImgboxHost(
+internal class ImgboxHost(
     httpService: HTTPService,
     dataTransaction: DataTransaction,
     downloadSpeedService: DownloadSpeedService,
 ) : Host("imgbox.com", 6, httpService, dataTransaction, downloadSpeedService) {
-    private val log by me.vripper.delegate.LoggerDelegate()
+    private val log by LoggerDelegate()
 
     @Throws(HostException::class)
     override fun resolve(

vripper-core/src/main/kotlin/me/vripper/host/ImxHost.kt

@@ -1,11 +1,14 @@
 package me.vripper.host
 
-import me.vripper.download.ImageDownloadContext
 import me.vripper.exception.HostException
 import me.vripper.exception.HtmlProcessorException
 import me.vripper.exception.XpathException
-import me.vripper.services.*
+import me.vripper.services.DataTransaction
+import me.vripper.services.DownloadService.ImageDownloadContext
+import me.vripper.services.DownloadSpeedService
+import me.vripper.services.HTTPService
 import me.vripper.utilities.HtmlUtils
+import me.vripper.utilities.LoggerDelegate
 import me.vripper.utilities.XpathUtils
 import org.apache.hc.client5.http.classic.methods.HttpPost
 import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
@@ -14,12 +17,12 @@ import org.w3c.dom.Document
 import org.w3c.dom.Node
 import java.io.IOException
 
-class ImxHost(
+internal class ImxHost(
     private val httpService: HTTPService,
     dataTransaction: DataTransaction,
     downloadSpeedService: DownloadSpeedService,
 ) : Host("imx.to", 8, httpService, dataTransaction, downloadSpeedService) {
-    private val log by me.vripper.delegate.LoggerDelegate()
+    private val log by LoggerDelegate()
 
     @Throws(HostException::class)
     override fun resolve(

vripper-core/src/main/kotlin/me/vripper/host/PimpandhostHost.kt

@@ -1,22 +1,25 @@
 package me.vripper.host
 
-import me.vripper.download.ImageDownloadContext
 import me.vripper.exception.HostException
 import me.vripper.exception.XpathException
-import me.vripper.services.*
+import me.vripper.services.DataTransaction
+import me.vripper.services.DownloadService.ImageDownloadContext
+import me.vripper.services.DownloadSpeedService
+import me.vripper.services.HTTPService
 import me.vripper.utilities.HtmlUtils
+import me.vripper.utilities.LoggerDelegate
 import me.vripper.utilities.XpathUtils
 import org.w3c.dom.Document
 import org.w3c.dom.Node
 import java.net.URI
 import java.net.URISyntaxException
 
-class PimpandhostHost(
+internal class PimpandhostHost(
     httpService: HTTPService,
     dataTransaction: DataTransaction,
     downloadSpeedService: DownloadSpeedService,
 ) : Host("pimpandhost.com", 9, httpService, dataTransaction, downloadSpeedService) {
-    private val log by me.vripper.delegate.LoggerDelegate()
+    private val log by LoggerDelegate()
 
     @Throws(HostException::class)
     override fun resolve(

vripper-core/src/main/kotlin/me/vripper/host/PixRouteHost.kt

@@ -1,19 +1,22 @@
 package me.vripper.host
 
-import me.vripper.download.ImageDownloadContext
 import me.vripper.exception.HostException
 import me.vripper.exception.XpathException
-import me.vripper.services.*
+import me.vripper.services.DataTransaction
+import me.vripper.services.DownloadService.ImageDownloadContext
+import me.vripper.services.DownloadSpeedService
+import me.vripper.services.HTTPService
+import me.vripper.utilities.LoggerDelegate
 import me.vripper.utilities.XpathUtils
 import org.w3c.dom.Document
 import org.w3c.dom.Node
 
-class PixRouteHost(
+internal class PixRouteHost(
     httpService: HTTPService,
     dataTransaction: DataTransaction,
     downloadSpeedService: DownloadSpeedService,
 ) : Host("pixroute.com", 11, httpService, dataTransaction, downloadSpeedService) {
-    private val log by me.vripper.delegate.LoggerDelegate()
+    private val log by LoggerDelegate()
 
     @Throws(HostException::class)
     override fun resolve(

vripper-core/src/main/kotlin/me/vripper/host/PixhostHost.kt

@@ -1,19 +1,22 @@
 package me.vripper.host
 
-import me.vripper.download.ImageDownloadContext
 import me.vripper.exception.HostException
 import me.vripper.exception.XpathException
-import me.vripper.services.*
+import me.vripper.services.DataTransaction
+import me.vripper.services.DownloadService.ImageDownloadContext
+import me.vripper.services.DownloadSpeedService
+import me.vripper.services.HTTPService
+import me.vripper.utilities.LoggerDelegate
 import me.vripper.utilities.XpathUtils
 import org.w3c.dom.Document
 import org.w3c.dom.Node
 
-class PixhostHost(
+internal class PixhostHost(
     httpService: HTTPService,
     dataTransaction: DataTransaction,
     downloadSpeedService: DownloadSpeedService,
 ) : Host("pixhost.to", 10, httpService, dataTransaction, downloadSpeedService) {
-    private val log by me.vripper.delegate.LoggerDelegate()
+    private val log by LoggerDelegate()
 
     @Throws(HostException::class)
     override fun resolve(

vripper-core/src/main/kotlin/me/vripper/host/PixxxelsHost.kt

@@ -1,18 +1,21 @@
 package me.vripper.host
 
-import me.vripper.download.ImageDownloadContext
 import me.vripper.exception.HostException
 import me.vripper.exception.XpathException
-import me.vripper.services.*
+import me.vripper.services.DataTransaction
+import me.vripper.services.DownloadService.ImageDownloadContext
+import me.vripper.services.DownloadSpeedService
+import me.vripper.services.HTTPService
+import me.vripper.utilities.LoggerDelegate
 import me.vripper.utilities.XpathUtils
 import org.w3c.dom.Document
 
-class PixxxelsHost(
+internal class PixxxelsHost(
     httpService: HTTPService,
     dataTransaction: DataTransaction,
     downloadSpeedService: DownloadSpeedService,
 ) : Host("pixxxels.cc", 12, httpService, dataTransaction, downloadSpeedService) {
-    private val log by me.vripper.delegate.LoggerDelegate()
+    private val log by LoggerDelegate()
 
     @Throws(HostException::class)
     override fun resolve(

vripper-core/src/main/kotlin/me/vripper/host/PostImgHost.kt

@@ -1,20 +1,23 @@
 package me.vripper.host
 
-import me.vripper.download.ImageDownloadContext
 import me.vripper.exception.HostException
 import me.vripper.exception.XpathException
-import me.vripper.services.*
+import me.vripper.services.DataTransaction
+import me.vripper.services.DownloadService.ImageDownloadContext
+import me.vripper.services.DownloadSpeedService
+import me.vripper.services.HTTPService
+import me.vripper.utilities.LoggerDelegate
 import me.vripper.utilities.XpathUtils
 import org.w3c.dom.Document
 import org.w3c.dom.Node
 import java.util.*
 
-class PostImgHost(
+internal class PostImgHost(
     httpService: HTTPService,
     dataTransaction: DataTransaction,
     downloadSpeedService: DownloadSpeedService,
 ) : Host("postimg.cc", 13, httpService, dataTransaction, downloadSpeedService) {
-    private val log by me.vripper.delegate.LoggerDelegate()
+    private val log by LoggerDelegate()
 
     @Throws(HostException::class)
     override fun resolve(

vripper-core/src/main/kotlin/me/vripper/host/TurboImageHost.kt

@@ -1,19 +1,22 @@
 package me.vripper.host
 
-import me.vripper.download.ImageDownloadContext
 import me.vripper.exception.HostException
 import me.vripper.exception.XpathException
-import me.vripper.services.*
+import me.vripper.services.DataTransaction
+import me.vripper.services.DownloadService.ImageDownloadContext
+import me.vripper.services.DownloadSpeedService
+import me.vripper.services.HTTPService
+import me.vripper.utilities.LoggerDelegate
 import me.vripper.utilities.XpathUtils
 import org.w3c.dom.Document
 import org.w3c.dom.Node
 
-class TurboImageHost(
+internal class TurboImageHost(
     httpService: HTTPService,
     dataTransaction: DataTransaction,
     downloadSpeedService: DownloadSpeedService,
 ) : Host("turboimagehost.com", 14, httpService, dataTransaction, downloadSpeedService) {
-    private val log by me.vripper.delegate.LoggerDelegate()
+    private val log by LoggerDelegate()
 
     @Throws(HostException::class)
     override fun resolve(

vripper-core/src/main/kotlin/me/vripper/host/ViprImHost.kt

@@ -1,20 +1,23 @@
 package me.vripper.host
 
-import me.vripper.download.ImageDownloadContext
 import me.vripper.exception.HostException
 import me.vripper.exception.XpathException
-import me.vripper.services.*
+import me.vripper.services.DataTransaction
+import me.vripper.services.DownloadService.ImageDownloadContext
+import me.vripper.services.DownloadSpeedService
+import me.vripper.services.HTTPService
+import me.vripper.utilities.LoggerDelegate
 import me.vripper.utilities.XpathUtils
 import org.w3c.dom.Document
 import org.w3c.dom.Node
 import java.util.*
 
-class ViprImHost(
+internal class ViprImHost(
     httpService: HTTPService,
     dataTransaction: DataTransaction,
     downloadSpeedService: DownloadSpeedService,
 ) : Host("vipr.im", 15, httpService, dataTransaction, downloadSpeedService) {
-    private val log by me.vripper.delegate.LoggerDelegate()
+    private val log by LoggerDelegate()
 
     @Throws(HostException::class)
     override fun resolve(

vripper-core/src/main/kotlin/me/vripper/listeners/AppManager.kt

@@ -0,0 +1,54 @@
+package me.vripper.listeners
+
+import kotlinx.coroutines.*
+import kotlinx.coroutines.flow.filterIsInstance
+import me.vripper.event.EventBus
+import me.vripper.event.SettingsUpdateEvent
+import me.vripper.services.*
+import org.koin.core.component.KoinComponent
+import org.koin.core.component.inject
+
+object AppManager : KoinComponent {
+    private val eventBus: EventBus by inject()
+    private val dataTransaction: DataTransaction by inject()
+    private val metadataService: MetadataService by inject()
+    private val settingsService: SettingsService by inject()
+    private val vgAuthService: VGAuthService by inject()
+    private val downloadSpeedService: DownloadSpeedService by inject()
+    private val httpService: HTTPService by inject()
+    private val retryPolicyService: RetryPolicyService by inject()
+    private val threadCacheService: ThreadCacheService by inject()
+    private val downloadService: DownloadService by inject()
+
+    private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
+    private var job: Job? = null
+
+    fun start() {
+        job?.cancel()
+        job = coroutineScope.launch {
+            eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collect {
+                httpService.client.close()
+                httpService.buildRequestConfig(it.settings.connectionSettings.timeout.toLong())
+                httpService.buildConnectionConfig(it.settings.connectionSettings.timeout.toLong())
+                httpService.buildConnectionPool()
+                httpService.buildClientBuilder()
+                vgAuthService.authenticate(it.settings)
+                retryPolicyService.maxAttempts = it.settings.connectionSettings.maxAttempts
+                threadCacheService.invalidate()
+            }
+        }
+        dataTransaction.setDownloadingToStopped()
+        dataTransaction.stopImagesByPostIdAndIsNotCompleted()
+        settingsService.init()
+        metadataService.fetchExisting()
+        downloadSpeedService.init()
+        downloadService.init()
+    }
+
+    fun stop() {
+        job?.cancel()
+        downloadService.halt()
+        downloadService.stop()
+        downloadSpeedService.halt()
+    }
+}

vripper-core/src/main/kotlin/me/vripper/listeners/OnStartupListener.kt

@@ -1,16 +0,0 @@
-package me.vripper.listeners
-
-import me.vripper.services.DataTransaction
-import me.vripper.services.MetadataService
-import org.koin.core.component.KoinComponent
-import org.koin.core.component.inject
-
-open class OnStartupListener : KoinComponent {
-    private val dataTransaction: DataTransaction by inject()
-    private val metadataService: MetadataService by inject()
-
-    open fun run() {
-        dataTransaction.setDownloadingToStopped()
-        metadataService.init()
-    }
-}

vripper-core/src/main/kotlin/me/vripper/model/DownloadSpeed.kt

@@ -1,3 +1,6 @@
 package me.vripper.model
 
+import kotlinx.serialization.Serializable
+
+@Serializable
 data class DownloadSpeed(val speed: Long)

vripper-core/src/main/kotlin/me/vripper/model/ErrorCount.kt

@@ -1,3 +1,6 @@
 package me.vripper.model
 
+import kotlinx.serialization.Serializable
+
+@Serializable
 data class ErrorCount(val count: Int)

vripper-core/src/main/kotlin/me/vripper/model/Post.kt

@@ -1,10 +1,13 @@
 package me.vripper.model
 
-import me.vripper.entities.domain.Status
+import kotlinx.serialization.Contextual
+import kotlinx.serialization.Serializable
+import me.vripper.entities.Status
 import java.time.LocalDateTime
 import kotlin.io.path.Path
 import kotlin.io.path.pathString
 
+@Serializable
 data class Post(
     val id: Long = -1,
     val postTitle: String,
@@ -17,7 +20,7 @@ data class Post(
     val total: Int,
     val hosts: Set<String>,
     val downloadDirectory: String,
-    val addedOn: LocalDateTime = LocalDateTime.now(),
+    @Contextual val addedOn: LocalDateTime = LocalDateTime.now(),
     var folderName: String,
     var status: Status = Status.STOPPED,
     var done: Int = 0,
@@ -28,7 +31,5 @@ data class Post(
     val postedBy: String,
     val resolvedNames: List<String>
 ) {
-    fun getDownloadFolder(): String {
-        return Path(downloadDirectory, folderName).pathString
-    }
+    fun getDownloadFolder() = Path(downloadDirectory, folderName).pathString
 }

vripper-core/src/main/kotlin/me/vripper/model/QueueState.kt

@@ -1,5 +1,8 @@
 package me.vripper.model
 
+import kotlinx.serialization.Serializable
+
+@Serializable
 data class QueueState(
     val running: Int,
     val remaining: Int

vripper-core/src/main/kotlin/me/vripper/model/Settings.kt

@@ -1,6 +1,9 @@
 package me.vripper.model
 
 import kotlinx.serialization.Serializable
+import me.vripper.utilities.ApplicationProperties.VRIPPER_DIR
+import java.nio.file.Files
+import kotlin.io.path.pathString
 
 @Serializable
 data class Settings(
@@ -17,11 +20,13 @@ data class ViperSettings(
     val password: String = "",
     val thanks: Boolean = false,
     val host: String = "https://vipergirls.to",
+    val requestLimit: Long = 2,
+    val fetchMetadata: Boolean = false,
 )
 
 @Serializable
 data class DownloadSettings(
-    val downloadPath: String = System.getProperty("user.home"),
+    val downloadPath: String = VRIPPER_DIR.resolve("downloads").also { Files.createDirectories(it) }.pathString,
     val autoStart: Boolean = true,
     val autoQueueThreshold: Int = 1,
     val forceOrder: Boolean = false,
@@ -41,7 +46,7 @@ data class ConnectionSettings(
 
 @Serializable
 data class SystemSettings(
-    val tempPath: String = System.getProperty("java.io.tmpdir"),
+    val tempPath: String = VRIPPER_DIR.resolve("temp").also { Files.createDirectories(it) }.pathString,
     val enableClipboardMonitoring: Boolean = false,
     val clipboardPollingRate: Int = 500,
     val maxEventLog: Int = 1_000,

vripper-core/src/main/kotlin/me/vripper/parser/ImageItem.kt

@@ -1,5 +0,0 @@
-package me.vripper.parser
-
-import me.vripper.host.Host
-
-data class ImageItem(val mainLink: String, val thumbLink: String, val host: Host)

vripper-core/src/main/kotlin/me/vripper/parser/PostLookupAPIParser.kt

@@ -1,74 +0,0 @@
-package me.vripper.parser
-
-import me.vripper.entities.LogEntryEntity
-import me.vripper.exception.DownloadException
-import me.vripper.exception.PostParseException
-import me.vripper.services.*
-import me.vripper.utilities.Tasks
-import me.vripper.utilities.formatToString
-import net.jodah.failsafe.Failsafe
-import net.jodah.failsafe.function.CheckedSupplier
-import org.apache.hc.client5.http.classic.HttpClient
-import org.apache.hc.client5.http.classic.methods.HttpGet
-import org.apache.hc.core5.net.URIBuilder
-import org.koin.core.component.KoinComponent
-import org.koin.core.component.inject
-import javax.xml.parsers.SAXParserFactory
-
-class PostLookupAPIParser(private val threadId: Long, private val postId: Long) : KoinComponent {
-    private val log by me.vripper.delegate.LoggerDelegate()
-    private val settingsService: SettingsService by inject()
-    private val retryPolicyService: RetryPolicyService by inject()
-    private val httpService: HTTPService by inject()
-    private val vgAuthService: VGAuthService by inject()
-    private val dataTransaction: DataTransaction by inject()
-
-    @Throws(PostParseException::class)
-    fun parse(): ThreadItem? {
-        log.debug("Parsing post $postId")
-        val httpGet =
-            HttpGet(URIBuilder("${settingsService.settings.viperSettings.host}/vr.php").also {
-                it.setParameter(
-                    "p", postId.toString()
-                )
-            }.build())
-        val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler()
-        log.debug("Requesting {}", httpGet)
-        Tasks.increment()
-        return try {
-            Failsafe.with(retryPolicyService.buildGenericRetryPolicy<Any>()).onFailure {
-                log.error(
-                    "parsing failed for thread $threadId, post $postId", it.failure
-                )
-                dataTransaction.saveLog(
-                    LogEntryEntity(
-                        type = LogEntryEntity.Type.POST, status = LogEntryEntity.Status.ERROR, message = """
-                    Failed to process thread $threadId, post $postId
-                    ${it.failure.formatToString()}
-                    """.trimIndent()
-                    )
-                )
-            }.get(CheckedSupplier {
-                val connection: HttpClient = httpService.client
-                connection.execute(
-                    httpGet, vgAuthService.context
-                ) { response ->
-                    if (response.code / 100 != 2) {
-                        throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
-                    }
-                    factory.newSAXParser()
-                        .parse(response.entity.content, threadLookupAPIResponseHandler)
-                    threadLookupAPIResponseHandler.result
-                }
-            })
-        } catch (e: Exception) {
-            throw PostParseException(e)
-        } finally {
-            Tasks.decrement()
-        }
-    }
-
-    companion object {
-        private val factory = SAXParserFactory.newInstance()
-    }
-}

vripper-core/src/main/kotlin/me/vripper/parser/ThreadLookupAPIParser.kt

@@ -1,78 +0,0 @@
-package me.vripper.parser
-
-import me.vripper.entities.LogEntryEntity
-import me.vripper.exception.DownloadException
-import me.vripper.exception.PostParseException
-import me.vripper.services.*
-import me.vripper.utilities.Tasks
-import me.vripper.utilities.formatToString
-import net.jodah.failsafe.Failsafe
-import net.jodah.failsafe.function.CheckedSupplier
-import org.apache.hc.client5.http.classic.methods.HttpGet
-import org.apache.hc.core5.net.URIBuilder
-import org.koin.core.component.KoinComponent
-import org.koin.core.component.inject
-import java.io.BufferedInputStream
-import java.util.*
-import javax.xml.parsers.SAXParserFactory
-
-class ThreadLookupAPIParser(private val threadId: Long) : KoinComponent {
-    private val log by me.vripper.delegate.LoggerDelegate()
-    private val cm: HTTPService by inject()
-    private val retryPolicyService: RetryPolicyService by inject()
-    private val vgAuthService: VGAuthService by inject()
-    private val settingsService: SettingsService by inject()
-    private val dataTransaction: DataTransaction by inject()
-
-    @Throws(PostParseException::class)
-    fun parse(): ThreadItem {
-        log.debug("Parsing thread $threadId")
-        val httpGet =
-            HttpGet(URIBuilder(settingsService.settings.viperSettings.host + "/vr.php").also {
-                it.setParameter(
-                    "t",
-                    threadId.toString()
-                )
-            }.build())
-        val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler()
-        log.debug("Requesting {}", httpGet)
-        Tasks.increment()
-        return try {
-            Failsafe.with(retryPolicyService.buildGenericRetryPolicy<Any>()).onFailure {
-                log.error(
-                    "parsing failed for thread $threadId",
-                    it.failure
-                )
-                dataTransaction.saveLog(
-                    LogEntryEntity(
-                        type = LogEntryEntity.Type.THREAD, status = LogEntryEntity.Status.ERROR, message = """
-                    Failed to process thread $threadId
-                    ${it.failure.formatToString()}
-                    """.trimIndent()
-                    )
-                )
-            }.get(CheckedSupplier {
-                cm.client.execute(
-                    httpGet, vgAuthService.context
-                ) { response ->
-                    if (response.code / 100 != 2) {
-                        throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
-                    }
-                    factory.newSAXParser().parse(
-                        BufferedInputStream(response.entity.content),
-                        threadLookupAPIResponseHandler
-                    )
-                    threadLookupAPIResponseHandler.result
-                }
-            })
-        } catch (e: Exception) {
-            throw PostParseException(e)
-        } finally {
-            Tasks.decrement()
-        }
-    }
-
-    companion object {
-        private val factory = SAXParserFactory.newInstance()
-    }
-}

vripper-core/src/main/kotlin/me/vripper/repositories/LogRepository.kt

@@ -1,14 +0,0 @@
-package me.vripper.repositories
-
-import me.vripper.entities.LogEntryEntity
-import java.util.*
-
-interface LogRepository {
-    fun save(logEntryEntity: LogEntryEntity): LogEntryEntity
-    fun update(logEntryEntity: LogEntryEntity)
-    fun findAll(): List<LogEntryEntity>
-    fun findById(id: Long): Optional<LogEntryEntity>
-
-    fun deleteOldest(): List<Long>
-    fun deleteAll()
-}

vripper-core/src/main/kotlin/me/vripper/repositories/impl/LogRepositoryImpl.kt

@@ -1,74 +0,0 @@
-package me.vripper.repositories.impl
-
-import me.vripper.entities.LogEntryEntity
-import me.vripper.repositories.LogRepository
-import me.vripper.services.SettingsService
-import me.vripper.tables.LogTable
-import org.jetbrains.exposed.sql.*
-import org.jetbrains.exposed.sql.SqlExpressionBuilder.notInSubQuery
-import java.util.*
-
-class LogRepositoryImpl(val settingsService: SettingsService) : LogRepository {
-
-    @Synchronized
-    override fun save(logEntryEntity: LogEntryEntity): LogEntryEntity {
-        val id = LogTable.insertAndGetId {
-            it[type] = logEntryEntity.type.name
-            it[status] = logEntryEntity.status.name
-            it[time] = logEntryEntity.time
-            it[message] = logEntryEntity.message.take(500)
-        }.value
-        return logEntryEntity.copy(id = id)
-    }
-
-    override fun update(logEntryEntity: LogEntryEntity) {
-        LogTable.update({ LogTable.id eq logEntryEntity.id }) {
-            it[type] = logEntryEntity.type.name
-            it[status] = logEntryEntity.status.name
-            it[message] = logEntryEntity.message.take(500)
-        }
-    }
-
-    override fun findById(id: Long): Optional<LogEntryEntity> {
-        val result = LogTable.select {
-            LogTable.id eq id
-        }.map(::transform)
-
-        return if (result.isEmpty()) {
-            Optional.empty()
-        } else {
-            Optional.of(result.first())
-        }
-    }
-
-    override fun findAll(): List<LogEntryEntity> {
-        return LogTable.selectAll().map(::transform)
-    }
-
-    override fun deleteOldest(): List<Long> {
-        val sub = LogTable.slice(LogTable.id).selectAll().orderBy(LogTable.time to SortOrder.DESC)
-            .limit(settingsService.settings.systemSettings.maxEventLog)
-        val ids = LogTable.select { LogTable.id notInSubQuery sub }.map { it[LogTable.id].value }
-        LogTable.deleteWhere { LogTable.id notInSubQuery sub }
-        return ids
-    }
-
-    override fun deleteAll() {
-        LogTable.deleteAll()
-    }
-
-    private fun transform(resultRow: ResultRow): LogEntryEntity {
-        val id = resultRow[LogTable.id].value
-        val type = LogEntryEntity.Type.valueOf(resultRow[LogTable.type])
-        val status = LogEntryEntity.Status.valueOf(resultRow[LogTable.status])
-        val time = resultRow[LogTable.time]
-        val message = resultRow[LogTable.message]
-        return LogEntryEntity(
-            id,
-            type,
-            status,
-            time,
-            message
-        )
-    }
-}

vripper-core/src/main/kotlin/me/vripper/services/AppEndpointService.kt

@@ -2,19 +2,21 @@ package me.vripper.services
 
 import kotlinx.coroutines.flow.*
 import kotlinx.coroutines.time.sample
-import me.vripper.download.DownloadService
-import me.vripper.entities.LogEntryEntity
-import me.vripper.entities.PostEntity
+import kotlinx.serialization.json.Json
+import me.vripper.entities.*
 import me.vripper.event.*
 import me.vripper.exception.PostParseException
 import me.vripper.model.*
-import me.vripper.tasks.AddPostRunnable
-import me.vripper.tasks.ThreadLookupRunnable
+import me.vripper.tasks.AddPostTask
+import me.vripper.tasks.ThreadLookupTask
 import me.vripper.utilities.ApplicationProperties
-import me.vripper.utilities.GLOBAL_EXECUTOR
+import me.vripper.utilities.ApplicationProperties.VRIPPER_DIR
+import me.vripper.utilities.LoggerDelegate
 import me.vripper.utilities.PathUtils
+import me.vripper.utilities.taskRunner
+import org.h2.jdbc.JdbcSQLNonTransientConnectionException
+import java.sql.DriverManager
 import java.time.Duration
-import java.util.concurrent.CompletableFuture
 import java.util.concurrent.locks.ReentrantLock
 import java.util.regex.Pattern
 import kotlin.concurrent.withLock
@@ -22,15 +24,16 @@ import kotlin.io.path.Path
 import kotlin.io.path.exists
 import kotlin.jvm.optionals.getOrNull
 
-class AppEndpointService(
+internal class AppEndpointService(
     private val downloadService: DownloadService,
     private val dataTransaction: DataTransaction,
     private val threadCacheService: ThreadCacheService,
     private val settingsService: SettingsService,
     private val vgAuthService: VGAuthService,
+    private val logService: LogService,
 ) : IAppEndpointService {
-    private val log by me.vripper.delegate.LoggerDelegate()
 
+    private val log by LoggerDelegate()
     private val lock = ReentrantLock()
 
     override suspend fun scanLinks(postLinks: String) {
@@ -38,10 +41,10 @@ class AppEndpointService(
             if (postLinks.isBlank()) {
                 return
             }
-            val urlList = postLinks.split(Pattern.compile("\\r?\\n")).dropLastWhile { it.isEmpty() }.map { it.trim() }
+            val urlList = postLinks.split(Pattern.compile("\\r?\\n")).dropLastWhile { it.isBlank() }.map { it.trim() }
                 .filter { it.isNotEmpty() }
             for (link in urlList) {
-                log.debug("Starting to process thread: $link")
+                log.debug("Scanning: $link")
                 if (!link.startsWith(settingsService.settings.viperSettings.host)) {
                     continue
                 }
@@ -54,27 +57,20 @@ class AppEndpointService(
                     threadId = m.group(1).toLong()
                     postId = m.group(4)?.toLong()
                     if (postId == null) {
-                        CompletableFuture.runAsync(
-                            ThreadLookupRunnable(
+                        taskRunner.submit(
+                            ThreadLookupTask(
                                 threadId, settingsService.settings
-                            ), GLOBAL_EXECUTOR
+                            )
                         )
                     } else {
-                        CompletableFuture.runAsync(
-                            AddPostRunnable(
+                        taskRunner.submit(
+                            AddPostTask(
                                 listOf(ThreadPostId(threadId, postId))
-                            ), GLOBAL_EXECUTOR
+                            )
                         )
                     }
                 } else {
-                    log.error("Cannot retrieve thread id from URL $link")
-                    dataTransaction.saveLog(
-                        LogEntryEntity(
-                            type = LogEntryEntity.Type.SCAN,
-                            status = LogEntryEntity.Status.ERROR,
-                            message = "Invalid link $link, link is missing the threadId"
-                        )
-                    )
+                    log.error("Invalid link $link, link is missing the threadId")
                     continue
                 }
             }
@@ -83,15 +79,13 @@ class AppEndpointService(
 
     override suspend fun restartAll(posIds: List<Long>) {
         lock.withLock {
-            downloadService.restartAll(posIds.map { dataTransaction.findPostByPostId(it) }.filter { it.isPresent }
-                .map { it.get() })
+            downloadService.restartAll(posIds.filter { dataTransaction.exists(it) }
+                .map { dataTransaction.findPostByPostId(it) })
         }
     }
 
     override suspend fun download(posts: List<ThreadPostId>) {
-        CompletableFuture.runAsync(
-            AddPostRunnable(posts), GLOBAL_EXECUTOR
-        )
+        taskRunner.submit(AddPostTask(posts))
     }
 
     override suspend fun stopAll(postIdList: List<Long>) {
@@ -124,6 +118,9 @@ class AppEndpointService(
                     )
                 }
                 val threadLookupResult = threadCacheService[thread.threadId]
+                if (threadLookupResult.error.isNotBlank()) {
+                    throw PostParseException("Error occurred for threadId = $threadId, ${threadLookupResult.error}")
+                }
                 threadLookupResult.postItemList.map { postItem ->
                     PostSelection(
                         postItem.threadId,
@@ -137,23 +134,13 @@ class AppEndpointService(
                         postItem.forum,
                         postItem.imageItemList.map { it.thumbLink })
                 }.ifEmpty {
-                    log.error(
-                        String.format(
-                            "Failed to get links for threadId = %s", threadId
-                        )
-                    )
                     throw PostParseException(
-                        String.format(
-                            "Failed to get links for threadId = %s", threadId
-                        )
+                        "Nothing found for threadId = $threadId"
                     )
                 }
             } catch (e: Exception) {
-                throw PostParseException(
-                    String.format(
-                        "Failed to get links for threadId = %s, %s", threadId, e.message
-                    )
-                )
+                log.error(e.message)
+                throw e
             }
         }
     }
@@ -172,12 +159,6 @@ class AppEndpointService(
         }
     }
 
-    override suspend fun logClear() {
-        lock.withLock {
-            dataTransaction.deleteAllLogs()
-        }
-    }
-
     override suspend fun renameToFirst(postIds: List<Long>) {
         postIds.forEach { postId ->
             dataTransaction
@@ -189,19 +170,24 @@ class AppEndpointService(
     }
 
     override suspend fun rename(postId: Long, newName: String) {
-        CompletableFuture.runAsync({
+        taskRunner.submit {
             synchronized(postId.toString().intern()) {
-                dataTransaction.findPostByPostId(postId).ifPresent { post ->
-                    if (Path(post.downloadDirectory, post.folderName).exists()) {
-                        PathUtils.rename(
-                            dataTransaction.findImagesByPostId(postId), post.downloadDirectory, post.folderName, newName
-                        )
+                if (dataTransaction.exists(postId)) {
+                    dataTransaction.findPostByPostId(postId).let { post ->
+                        if (Path(post.downloadDirectory, post.folderName).exists()) {
+                            PathUtils.rename(
+                                dataTransaction.findImagesByPostId(postId),
+                                post.downloadDirectory,
+                                post.folderName,
+                                newName
+                            )
+                        }
+                        post.folderName = PathUtils.sanitize(newName)
+                        dataTransaction.updatePost(post)
                     }
-                    post.folderName = PathUtils.sanitize(newName)
-                    dataTransaction.updatePost(post)
                 }
             }
-        }, GLOBAL_EXECUTOR)
+        }
     }
 
     override fun onNewPosts(): Flow<Post> =
@@ -255,38 +241,27 @@ class AppEndpointService(
     }
 
     override suspend fun findPost(postId: Long): Post {
-        return mapper(dataTransaction.findPostByPostId(postId).orElseThrow())
+        return mapper(dataTransaction.findPostByPostId(postId))
     }
 
     override suspend fun findImagesByPostId(postId: Long): List<Image> {
         return dataTransaction.findImagesByPostId(postId)
     }
 
-    override fun onUpdateImages(postId: Long): Flow<Image> = EventBus.events.filterIsInstance(ImageEvent::class).map {
-        it.imageEntities.filter { imageEntity: Image -> imageEntity.postId == postId }
-    }.filter { it.isNotEmpty() }.flatMapConcat { it.asFlow() }
+    override fun onUpdateImagesByPostId(postId: Long): Flow<Image> =
+        EventBus.events.filterIsInstance(ImageEvent::class).map {
+            it.imageEntities.filter { imageEntity: Image -> imageEntity.postId == postId }
+        }.filter { it.isNotEmpty() }.flatMapConcat { it.asFlow() }
 
+    override fun onUpdateImages(): Flow<Image> =
+        EventBus.events.filterIsInstance(ImageEvent::class).flatMapConcat { it.imageEntities.asFlow() }
 
     override fun onStopped(): Flow<Long> =
         EventBus.events.filterIsInstance(StoppedEvent::class).flatMapConcat { it.postIds.asFlow() }
 
-
-    override suspend fun findAllLogs(): List<LogEntry> {
-        return dataTransaction.findAllLogs()
-    }
-
     override fun onNewLog() = EventBus.events.filterIsInstance(LogCreateEvent::class).map { it.logEntryEntity }
 
-
-    override fun onUpdateLog() = EventBus.events.filterIsInstance(LogUpdateEvent::class).map {
-        it.logEntryEntity
-    }
-
-
-    override fun onDeleteLogs(): Flow<Long> = EventBus.events.filterIsInstance(LogDeleteEvent::class).flatMapConcat {
-        it.deleted.asFlow()
-    }
-
+    override suspend fun initLogger() = logService.init()
 
     override fun onNewThread(): Flow<Thread> =
         EventBus.events.filterIsInstance(ThreadCreateEvent::class).map { it.threadEntity }
@@ -307,8 +282,8 @@ class AppEndpointService(
         return dataTransaction.findAllThreads()
     }
 
-    override fun onDownloadSpeed(): Flow<Long> =
-        EventBus.events.filterIsInstance(DownloadSpeedEvent::class).map { it.downloadSpeed.speed }
+    override fun onDownloadSpeed(): Flow<DownloadSpeed> =
+        EventBus.events.filterIsInstance(DownloadSpeedEvent::class).map { it.downloadSpeed }
 
     override fun onVGUserUpdate(): Flow<String> =
         EventBus.events.filterIsInstance(VGUserLoginEvent::class).map { it.username }
@@ -316,8 +291,8 @@ class AppEndpointService(
     override fun onQueueStateUpdate(): Flow<QueueState> =
         EventBus.events.filterIsInstance(QueueStateEvent::class).map { it.queueState }
 
-    override fun onErrorCountUpdate(): Flow<Int> =
-        EventBus.events.filterIsInstance(ErrorCountEvent::class).map { it.errorCount.count }
+    override fun onErrorCountUpdate(): Flow<ErrorCount> =
+        EventBus.events.filterIsInstance(ErrorCountEvent::class).map { it.errorCount }
 
     override fun onTasksRunning(): Flow<Boolean> =
         EventBus.events.filterIsInstance(LoadingTasks::class).sample(Duration.ofMillis(500)).map { it.loading }
@@ -335,4 +310,149 @@ class AppEndpointService(
 
     override suspend fun getVersion(): String = ApplicationProperties.VERSION
 
+    override suspend fun dbMigration(): String {
+
+        val conn = try {
+            DriverManager
+                .getConnection("jdbc:h2:file:$VRIPPER_DIR/vripper;DB_CLOSE_DELAY=-1;IFEXISTS=TRUE")
+        } catch (_: JdbcSQLNonTransientConnectionException) {
+            return "Old database not found, nothing to do"
+        }
+
+        var postsCount = 0;
+        var threadCount = 0;
+
+        conn.use { conn ->
+            conn.prepareStatement("select * from post").use {
+                it.executeQuery().use {
+                    while (it.next()) {
+                        val done = it.getInt("DONE")
+                        val hosts = it.getString("HOSTS")
+                        val outputPath = it.getString("OUTPUT_PATH")
+                        val postId = it.getLong("POST_ID")
+                        val status = it.getString("STATUS")
+                        val threadId = it.getLong("THREAD_ID")
+                        val postTitle = it.getString("POST_TITLE")
+                        val threadTitle = it.getString("THREAD_TITLE")
+                        val forum = it.getString("FORUM")
+                        val total = it.getInt("TOTAL")
+                        val size = it.getLong("SIZE")
+                        val downloaded = it.getLong("DOWNLOADED")
+                        val url = it.getString("URL")
+                        val token = it.getString("TOKEN")
+                        val addedAt = it.getTimestamp("ADDED_AT")
+                        val folderName = it.getString("FOLDER_NAME") ?: ""
+
+                        val exists = dataTransaction.exists(postId)
+                        if (exists) {
+                            continue
+                        }
+
+                        val post = PostEntity(
+                            postTitle = postTitle,
+                            threadTitle = threadTitle,
+                            forum = forum,
+                            url = url,
+                            token = token,
+                            postId = postId,
+                            threadId = threadId,
+                            total = total,
+                            hosts = hosts.split(";").dropLastWhile { it.isEmpty() }.toSet(),
+                            downloadDirectory = outputPath,
+                            addedOn = addedAt.toLocalDateTime(),
+                            folderName = folderName,
+                            status = Status.valueOf(status),
+                            done = done,
+                            size = size,
+                            downloaded = downloaded,
+                        )
+
+                        val images = mutableListOf<ImageEntity>()
+
+                        // load images
+                        conn.prepareStatement("select * from image where post_id = ?").use { statement ->
+                            statement.setLong(1, postId)
+                            statement.executeQuery().use { set ->
+                                while (set.next()) {
+                                    val downloaded = set.getLong("DOWNLOADED")
+                                    val host = set.getByte("HOST")
+                                    val index = set.getInt("INDEX")
+                                    val status = Status.valueOf(set.getString("STATUS"))
+                                    val size = set.getLong("SIZE")
+                                    val url = set.getString("URL")
+                                    val thumbUrl = set.getString("THUMB_URL")
+                                    val fileName = set.getString("FILENAME") ?: ""
+
+                                    ImageEntity(
+                                        postId = postId,
+                                        url = url,
+                                        thumbUrl = thumbUrl,
+                                        host = host,
+                                        index = index,
+                                        size = size,
+                                        downloaded = downloaded,
+                                        status = status,
+                                        filename = fileName,
+                                    ).also { images.add(it) }
+                                }
+                            }
+                        }
+
+                        dataTransaction.saveAndNotify(post, images)
+
+                        //load meta
+                        conn.prepareStatement("select * from metadata where post_id = ?").use { statement ->
+                            statement.setLong(1, postId)
+                            statement.executeQuery().use { set ->
+                                if (set.next()) {
+                                    val data = Json.decodeFromString(set.getString("DATA")) as MetadataEntity.Data
+                                    val metadata = MetadataEntity(
+                                        postId = postId,
+                                        data = data
+                                    )
+                                    dataTransaction.saveMetadata(metadata)
+                                }
+                            }
+                        }
+                        postsCount++
+                    }
+                }
+            }
+
+            conn.prepareStatement("select * from thread").use { statement ->
+                statement.executeQuery().use { set ->
+                    while (set.next()) {
+                        val total = set.getInt("TOTAL")
+                        val url = set.getString("URL")
+                        val threadId = set.getLong("THREAD_ID")
+                        val title = set.getString("TITLE")
+
+                        if (dataTransaction.findThreadByThreadId(threadId).isPresent) {
+                            continue
+                        }
+
+                        val threadEntity = ThreadEntity(
+                            title = title,
+                            link = url,
+                            threadId = threadId,
+                            total = total,
+                        )
+
+                        dataTransaction.save(threadEntity)
+                        threadCount++
+                    }
+                }
+            }
+        }
+
+        return if (postsCount == 0 && threadCount == 0) {
+            "Nothing to do, everything was imported"
+        } else {
+            "Successfully imported $postsCount posts and $threadCount threads"
+        }
+    }
+
+    override fun connectionState(): String {
+        return ""
+    }
 }

vripper-core/src/main/kotlin/me/vripper/services/DataTransaction.kt

@@ -1,89 +1,106 @@
 package me.vripper.services
 
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.SupervisorJob
-import kotlinx.coroutines.launch
+import com.github.benmanes.caffeine.cache.Caffeine
+import com.github.benmanes.caffeine.cache.LoadingCache
+import me.vripper.data.repositories.ImageRepository
+import me.vripper.data.repositories.MetadataRepository
+import me.vripper.data.repositories.PostRepository
+import me.vripper.data.repositories.ThreadRepository
 import me.vripper.entities.*
-import me.vripper.entities.domain.Status
 import me.vripper.event.*
 import me.vripper.model.ErrorCount
-import me.vripper.parser.PostItem
-import me.vripper.repositories.*
 import me.vripper.utilities.PathUtils
 import me.vripper.utilities.PathUtils.sanitize
+import me.vripper.vgapi.PostItem
 import org.jetbrains.exposed.sql.transactions.transaction
 import java.util.*
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicInteger
 import kotlin.io.path.pathString
 
-class DataTransaction(
+internal class DataTransaction(
     private val settingsService: SettingsService,
-    private val postDownloadStateRepository: PostDownloadStateRepository,
+    private val postRepository: PostRepository,
     private val imageRepository: ImageRepository,
     private val threadRepository: ThreadRepository,
     private val metadataRepository: MetadataRepository,
-    private val logRepository: LogRepository,
     private val eventBus: EventBus,
 ) {
 
-    private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
+    private val nextRank = AtomicInteger(transaction { getQueuePosition() }?.plus(1) ?: 0)
+    private val postEntityIdCache: LoadingCache<Long, PostEntity> =
+        Caffeine.newBuilder().expireAfterAccess(5, TimeUnit.MINUTES).build { id ->
+            transaction { postRepository.findById(id) }
+        }
+
+    private val postPostIdCache: LoadingCache<Long, PostEntity> =
+        Caffeine.newBuilder().expireAfterAccess(5, TimeUnit.MINUTES).build { id ->
+            transaction { postRepository.findByPostId(id) }
+        }
 
     private fun save(postEntities: List<PostEntity>): List<PostEntity> {
-        return transaction { postDownloadStateRepository.save(postEntities) }
+        return transaction { postRepository.save(postEntities) }
+    }
+
+    fun saveAndNotify(postEntity: PostEntity, images: List<ImageEntity>) {
+        val savedPost = transaction {
+            val savedPost =
+                postRepository.save(listOf(postEntity.copy(rank = nextRank.andIncrement))).first()
+            save(images.map { it.copy(postIdRef = savedPost.id) })
+            savedPost
+        }
+        eventBus.publishEvent(PostCreateEvent(listOf(savedPost)))
     }
 
     fun updatePosts(postEntities: List<PostEntity>) {
-        transaction { postDownloadStateRepository.update(postEntities) }
-        coroutineScope.launch {
-            eventBus.publishEvent(PostUpdateEvent(postEntities))
+        transaction { postRepository.update(postEntities) }
+        postEntities.forEach { postEntity ->
+            postPostIdCache.put(postEntity.postId, postEntity)
+            postEntityIdCache.put(postEntity.id, postEntity)
         }
+        eventBus.publishEvent(PostUpdateEvent(postEntities))
     }
 
     fun updatePost(postEntity: PostEntity) {
-        transaction { postDownloadStateRepository.update(postEntity) }
-        coroutineScope.launch {
-            eventBus.publishEvent(PostUpdateEvent(listOf(postEntity)))
-        }
+        transaction { postRepository.update(postEntity) }
+        postPostIdCache.put(postEntity.postId, postEntity)
+        postEntityIdCache.put(postEntity.id, postEntity)
+        eventBus.publishEvent(PostUpdateEvent(listOf(postEntity)))
     }
 
     fun save(threadEntity: ThreadEntity) {
         val savedThread = transaction { threadRepository.save(threadEntity) }
-        coroutineScope.launch {
-            eventBus.publishEvent(ThreadCreateEvent(savedThread))
-        }
+        eventBus.publishEvent(ThreadCreateEvent(savedThread))
     }
 
     fun update(threadEntity: ThreadEntity) {
         transaction { threadRepository.update(threadEntity) }
-        coroutineScope.launch {
-            eventBus.publishEvent(ThreadUpdateEvent(threadEntity))
-        }
+        eventBus.publishEvent(ThreadUpdateEvent(threadEntity))
     }
 
     fun updateImages(imageEntities: List<ImageEntity>) {
         transaction { imageRepository.update(imageEntities) }
-        coroutineScope.launch {
-            eventBus.publishEvent(ImageEvent(imageEntities))
-        }
+        eventBus.publishEvent(ImageEvent(imageEntities))
     }
 
-    fun updateImage(imageEntity: ImageEntity) {
-        transaction { imageRepository.update(imageEntity) }
-        coroutineScope.launch {
-            eventBus.publishEvent(ImageEvent(listOf(imageEntity)))
+    fun updateImage(imageEntity: ImageEntity, persist: Boolean = true) {
+        if (persist) {
+            transaction {
+                imageRepository.update(imageEntity)
+            }
         }
+        eventBus.publishEvent(ImageEvent(listOf(imageEntity)))
     }
 
     fun exists(postId: Long): Boolean {
-        return transaction { postDownloadStateRepository.existByPostId(postId) }
+        if (postPostIdCache.getIfPresent(postId) != null) {
+            return true
+        }
+        return transaction { postRepository.existByPostId(postId) }
     }
 
     @Synchronized
     fun newPosts(postItems: List<PostItem>): List<PostEntity> {
-        var queuePosition = transaction {
-            getQueuePosition()?.plus(1) ?: 0
-        }
-
         val posts = postItems.associate { postItem ->
             val postEntity = PostEntity(
                 postTitle = postItem.title,
@@ -95,7 +112,7 @@ class DataTransaction(
                 hosts = postItem.hosts.map { "${it.first} (${it.second})" }.toSet(),
                 threadTitle = postItem.threadTitle,
                 forum = postItem.forum,
-                rank = queuePosition++,
+                rank = nextRank.andIncrement,
                 downloadDirectory = PathUtils.calculateDownloadPath(
                     postItem.forum,
                     postItem.threadTitle,
@@ -127,14 +144,12 @@ class DataTransaction(
             }
             savedPosts
         }
-        coroutineScope.launch {
-            eventBus.publishEvent(PostCreateEvent(savedPosts))
-        }
+        eventBus.publishEvent(PostCreateEvent(savedPosts))
         return savedPosts
     }
 
     private fun getQueuePosition(): Int? {
-        return postDownloadStateRepository.findMaxRank()
+        return postRepository.findMaxRank()
     }
 
     private fun save(imageEntities: List<ImageEntity>) {
@@ -142,7 +157,7 @@ class DataTransaction(
     }
 
     fun finishPost(postId: Long, automatic: Boolean = false) {
-        val post = findPostByPostId(postId).orElseThrow()
+        val post = findPostByPostId(postId)
         val imagesInErrorStatus = findByPostIdAndIsError(post.postId)
         if (imagesInErrorStatus.isNotEmpty()) {
             post.status = Status.ERROR
@@ -173,27 +188,24 @@ class DataTransaction(
         transaction {
             metadataRepository.deleteAllByPostId(postIds)
             imageRepository.deleteAllByPostId(postIds)
-            postDownloadStateRepository.deleteAll(postIds)
+            postRepository.deleteAll(postIds)
             sortPostsByRank()
         }
-
-        coroutineScope.launch {
-            eventBus.publishEvent(PostDeleteEvent(postIds = postIds))
-        }
-        coroutineScope.launch {
-            eventBus.publishEvent(ErrorCountEvent(ErrorCount(countImagesInError())))
+        postIds.forEach { postId ->
+            postPostIdCache.get(postId)?.let { postEntityIdCache.invalidate(it.id) }
+            postPostIdCache.invalidate(postId)
         }
+        eventBus.publishEvent(PostDeleteEvent(postIds = postIds))
+        eventBus.publishEvent(ErrorCountEvent(ErrorCount(countImagesInError())))
     }
 
     fun removeThread(threadId: Long) {
         transaction { threadRepository.deleteByThreadId(threadId) }
-        coroutineScope.launch {
-            eventBus.publishEvent(ThreadDeleteEvent(threadId))
-        }
+        eventBus.publishEvent(ThreadDeleteEvent(threadId))
     }
 
     fun clearCompleted(): List<Long> {
-        val completed = transaction { postDownloadStateRepository.findCompleted() }
+        val completed = transaction { postRepository.findCompleted() }
         remove(completed)
         return completed
     }
@@ -210,39 +222,43 @@ class DataTransaction(
         transaction { imageRepository.stopByPostIdAndIsNotCompleted(postId) }
     }
 
+    fun stopImagesByPostIdAndIsNotCompleted() {
+        transaction { imageRepository.stopByPostIdAndIsNotCompleted() }
+    }
+
     fun saveMetadata(metadataEntity: MetadataEntity) {
         transaction { metadataRepository.save(metadataEntity) }
-        coroutineScope.launch {
-            eventBus.publishEvent(MetadataUpdateEvent(metadataEntity))
-        }
+        eventBus.publishEvent(MetadataUpdateEvent(metadataEntity))
     }
 
     fun clearQueueLinks() {
         transaction { threadRepository.deleteAll() }
-        coroutineScope.launch {
-            eventBus.publishEvent(ThreadClearEvent())
-        }
+        eventBus.publishEvent(ThreadClearEvent())
     }
 
     @Synchronized
     fun sortPostsByRank() {
-        val postEntities = findAllPosts().sortedWith(Comparator.comparing(PostEntity::addedOn))
+        val postsToUpdate = mutableListOf<PostEntity>()
+        val postEntities = findAllPosts().sortedWith(Comparator.comparing(PostEntity::rank))
         for (i in postEntities.indices) {
-            postEntities[i].rank = i
+            if (postEntities[i].rank != i) {
+                postsToUpdate.add(postEntities[i].copy(rank = i))
+            }
         }
-        updatePosts(postEntities)
+        updatePosts(postsToUpdate)
+        nextRank.set(transaction { getQueuePosition() }?.plus(1) ?: 0)
     }
 
     fun setDownloadingToStopped() {
-        transaction { postDownloadStateRepository.setDownloadingToStopped() }
+        transaction { postRepository.setDownloadingToStopped() }
     }
 
     fun findAllPosts(): List<PostEntity> {
-        return transaction { postDownloadStateRepository.findAll() }
+        return transaction { postRepository.findAll() }
     }
 
-    fun findPostById(id: Long): Optional<PostEntity> {
-        return transaction { postDownloadStateRepository.findById(id) }
+    fun findPostById(id: Long): PostEntity {
+        return postEntityIdCache.get(id) ?: throw NoSuchElementException("Post with id = $id does not exist")
     }
 
     fun findImagesByPostId(postId: Long): List<ImageEntity> {
@@ -271,46 +287,16 @@ class DataTransaction(
         return transaction { imageRepository.countError() }
     }
 
-    fun findPostByPostId(postId: Long): Optional<PostEntity> {
-        return transaction { postDownloadStateRepository.findByPostId(postId) }
+    fun findPostByPostId(postId: Long): PostEntity {
+        return postPostIdCache.get(postId) ?: throw NoSuchElementException("Post with postId = $postId does not exist")
     }
 
     fun findThreadByThreadId(threadId: Long): Optional<ThreadEntity> {
         return transaction { threadRepository.findByThreadId(threadId) }
     }
 
-    fun saveLog(logEntryEntity: LogEntryEntity): LogEntryEntity {
-        val pair = transaction {
-            val saved = logRepository.save(logEntryEntity)
-            val deleted = logRepository.deleteOldest()
-            Pair(saved, deleted)
-        }
-        coroutineScope.launch {
-            eventBus.publishEvent(LogCreateEvent(pair.first))
-        }
-        coroutineScope.launch {
-            eventBus.publishEvent(LogDeleteEvent(pair.second))
-        }
-        return pair.first
-    }
-
-    fun updateLog(logEntryEntity: LogEntryEntity) {
-        transaction { logRepository.update(logEntryEntity) }
-        coroutineScope.launch {
-            eventBus.publishEvent(LogUpdateEvent(logEntryEntity))
-        }
-    }
-
-    fun deleteAllLogs() {
-        transaction { logRepository.deleteAll() }
-    }
-
-    fun findAllLogs(): List<LogEntryEntity> {
-        return transaction { logRepository.findAll() }
-    }
-
     fun findAllNonCompletedPostIds(): List<Long> {
-        return transaction { postDownloadStateRepository.findAllNonCompletedPostIds() }
+        return transaction { postRepository.findAllNonCompletedPostIds() }
     }
 
     fun findMetadataByPostId(postId: Long): Optional<MetadataEntity> {

vripper-core/src/main/kotlin/me/vripper/services/DownloadService.kt

@@ -0,0 +1,451 @@
+package me.vripper.services
+
+import dev.failsafe.Failsafe
+import dev.failsafe.RetryPolicy
+import dev.failsafe.function.CheckedRunnable
+import kotlinx.coroutines.*
+import me.vripper.entities.ImageEntity
+import me.vripper.entities.PostEntity
+import me.vripper.entities.Status
+import me.vripper.event.ErrorCountEvent
+import me.vripper.event.EventBus
+import me.vripper.event.QueueStateEvent
+import me.vripper.event.StoppedEvent
+import me.vripper.exception.DownloadException
+import me.vripper.exception.HostException
+import me.vripper.host.DownloadedImage
+import me.vripper.host.Host
+import me.vripper.host.ImageMimeType
+import me.vripper.model.ErrorCount
+import me.vripper.model.QueueState
+import me.vripper.model.Settings
+import me.vripper.utilities.LoggerDelegate
+import me.vripper.utilities.PathUtils.getExtension
+import me.vripper.utilities.PathUtils.getFileNameWithoutExtension
+import me.vripper.utilities.PathUtils.sanitize
+import me.vripper.utilities.downloadRunner
+import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase
+import org.apache.hc.client5.http.cookie.BasicCookieStore
+import org.apache.hc.client5.http.protocol.HttpClientContext
+import org.jetbrains.exposed.sql.transactions.transaction
+import org.koin.core.component.KoinComponent
+import org.koin.core.component.inject
+import java.io.IOException
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.StandardCopyOption
+import java.util.*
+import java.util.concurrent.locks.ReentrantLock
+import kotlin.concurrent.withLock
+import kotlin.io.path.Path
+import kotlin.io.path.pathString
+
+internal class DownloadService(
+    private val settingsService: SettingsService,
+    private val dataTransaction: DataTransaction,
+    private val retryPolicyService: RetryPolicyService,
+    private val eventBus: EventBus
+) {
+    private val maxPoolSize: Int = 24
+    private val log by LoggerDelegate()
+    private val running: MutableMap<Byte, MutableList<ImageDownloadRunnable>> = mutableMapOf()
+    private val pending: MutableMap<Byte, MutableList<ImageDownloadRunnable>> = mutableMapOf()
+    private val lock = ReentrantLock()
+    private val condition = lock.newCondition()
+
+    private var downloadMonitorThread: Thread? = null
+
+    internal class ImageDownloadContext(val imageEntity: ImageEntity, val settings: Settings) : KoinComponent {
+        private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
+        private val jobs = mutableListOf<Job>()
+        val httpContext: HttpClientContext =
+            HttpClientContext.create().apply { cookieStore = BasicCookieStore() }
+        val requests = mutableListOf<HttpUriRequestBase>()
+        val postId = imageEntity.postIdRef
+
+        fun cancelCoroutines() {
+            runBlocking {
+                coroutineScope.cancel()
+                jobs.forEach { job -> job.cancelAndJoin() }
+            }
+        }
+
+        fun launchCoroutine(block: suspend CoroutineScope.() -> Unit): Job {
+            return coroutineScope.launch(block = block).also { job -> jobs.add(job) }
+        }
+    }
+
+    internal class ImageDownloadRunnable(
+        val imageEntity: ImageEntity, val postRank: Int, private val settings: Settings
+    ) : KoinComponent, CheckedRunnable {
+        private val log by LoggerDelegate()
+        private val dataTransaction: DataTransaction by inject()
+        private val vgauthService: VGAuthService by inject()
+        private val hosts: List<Host> = getKoin().getAll()
+        var completed = false
+        var stopped = false
+
+        private lateinit var context: ImageDownloadContext
+
+        fun download() {
+            try {
+                imageEntity.status = Status.DOWNLOADING
+                imageEntity.downloaded = 0
+                dataTransaction.updateImage(imageEntity)
+                synchronized(imageEntity.postId.toString().intern()) {
+                    val post = dataTransaction.findPostById(context.postId)
+                    if (post.status != Status.DOWNLOADING) {
+                        post.status = Status.DOWNLOADING
+                        dataTransaction.updatePost(post)
+                        vgauthService.leaveThanks(post)
+                    }
+                }
+                log.debug("Getting image url and name from ${imageEntity.url} using ${imageEntity.host}")
+                val host = hosts.first { it.isSupported(imageEntity.url) }
+                val downloadedImage = host.downloadInternal(imageEntity.url, context)
+                log.debug("Resolved name for ${imageEntity.url}: ${downloadedImage.name}")
+                log.debug("Downloaded image {} to {}", imageEntity.url, downloadedImage.path)
+                synchronized(imageEntity.postId.toString().intern()) {
+                    val post = dataTransaction.findPostById(context.postId)
+                    val downloadDirectory = Path(post.downloadDirectory, post.folderName).pathString
+                    checkImageTypeAndRename(
+                        downloadDirectory, downloadedImage, imageEntity.index
+                    )
+                    if (imageEntity.downloaded == imageEntity.size && imageEntity.size > 0) {
+                        imageEntity.status = Status.FINISHED
+                        post.done += 1
+                        post.downloaded += imageEntity.size
+                        dataTransaction.updatePost(post)
+                    } else {
+                        imageEntity.status = Status.ERROR
+                    }
+                    dataTransaction.updateImage(imageEntity)
+                }
+            } catch (e: Exception) {
+                if (stopped) {
+                    return
+                }
+                imageEntity.status = Status.ERROR
+                dataTransaction.updateImage(imageEntity)
+                throw DownloadException(e)
+            }
+        }
+
+        @Throws(HostException::class)
+        private fun checkImageTypeAndRename(
+            downloadDirectory: String, downloadedImage: DownloadedImage, index: Int
+        ) {
+            val existingExtension = getExtension(downloadedImage.name).lowercase()
+            val fileNameWithoutExtension = getFileNameWithoutExtension(downloadedImage.name)
+            val extension = when (downloadedImage.type) {
+                ImageMimeType.IMAGE_BMP -> "BMP"
+                ImageMimeType.IMAGE_GIF -> "GIF"
+                ImageMimeType.IMAGE_JPEG -> "JPG"
+                ImageMimeType.IMAGE_PNG -> "PNG"
+                ImageMimeType.IMAGE_WEBP -> "WEBP"
+            }
+            val filename =
+                if (existingExtension.isBlank()) "${sanitize(downloadedImage.name)}.$extension" else "${
+                    sanitize(
+                        fileNameWithoutExtension
+                    )
+                }.$extension"
+            try {
+                val downloadDestinationFolder = Path.of(downloadDirectory)
+                Files.createDirectories(downloadDestinationFolder)
+                val finalFilename = "${
+                    if (settings.downloadSettings.forceOrder) String.format(
+                        "%03d_", index + 1
+                    ) else ""
+                }$filename"
+                imageEntity.filename = finalFilename
+                val imageDownloadPath = downloadDestinationFolder.resolve(finalFilename)
+                Files.copy(downloadedImage.path, imageDownloadPath, StandardCopyOption.REPLACE_EXISTING)
+            } catch (e: Exception) {
+                throw HostException("Failed to rename the image", e)
+            } finally {
+                try {
+                    Files.delete(downloadedImage.path)
+                } catch (_: IOException) {
+                }
+            }
+        }
+
+        override fun run() {
+            context = ImageDownloadContext(imageEntity, settings)
+            try {
+                if (stopped) {
+                    return
+                }
+                download()
+            } finally {
+                completed = true
+                context.cancelCoroutines()
+            }
+        }
+
+        fun stop() {
+            stopped = true
+            context.requests.forEach { it.abort() }
+            context.cancelCoroutines()
+            dataTransaction.updateImage(context.imageEntity)
+        }
+
+        override fun equals(other: Any?): Boolean {
+            if (this === other) return true
+            if (other == null || javaClass != other.javaClass) return false
+            val that = other as ImageDownloadRunnable
+            return imageEntity.id == that.imageEntity.id
+        }
+
+        override fun hashCode(): Int {
+            return Objects.hash(imageEntity.id)
+        }
+    }
+
+    fun init() {
+        downloadMonitorThread?.interrupt()
+        downloadMonitorThread = Thread.ofVirtual().name("Download Monitor").unstarted(Runnable {
+            log.info("Scheduler have been initialized")
+            val accepted: MutableList<ImageDownloadRunnable> = mutableListOf()
+            val candidates: MutableList<ImageDownloadRunnable> = mutableListOf()
+            while (!Thread.currentThread().isInterrupted) {
+                lock.withLock {
+                    candidates.addAll(getCandidates(candidateCount()))
+                    candidates.forEach {
+                        if (canRun(it.imageEntity.host)) {
+                            accepted.add(it)
+                            running[it.imageEntity.host]!!.add(it)
+                            log.debug("${it.imageEntity.url} accepted to run")
+                        }
+                    }
+                    accepted.forEach {
+                        pending[it.imageEntity.host]?.remove(it)
+                        scheduleForDownload(it)
+                    }
+                    accepted.clear()
+                    candidates.clear()
+                    try {
+                        condition.await()
+                    } catch (_: InterruptedException) {
+                        Thread.currentThread().interrupt()
+                    }
+                }
+            }
+            log.info("Scheduler have been shutdown")
+        })
+        downloadMonitorThread?.start()
+    }
+
+    fun halt() {
+        downloadMonitorThread?.interrupt()
+    }
+
+    fun stop(postIds: List<Long> = emptyList()) {
+        if (postIds.isNotEmpty()) {
+            stopInternal(postIds)
+            eventBus.publishEvent(StoppedEvent(postIds))
+        } else {
+            stopAll()
+            eventBus.publishEvent(StoppedEvent(listOf(-1)))
+        }
+    }
+
+    fun restartAll(postEntityIds: List<PostEntity> = emptyList()) {
+        if (postEntityIds.isNotEmpty()) {
+            restart(postEntityIds.associateWith { dataTransaction.findByPostIdAndIsNotCompleted(it.postId) })
+        } else {
+            restart(
+                dataTransaction.findAllPosts()
+                    .associateWith { dataTransaction.findByPostIdAndIsNotCompleted(it.postId) })
+        }
+    }
+
+    private fun restart(posts: Map<PostEntity, List<ImageEntity>>) {
+        lock.withLock {
+            val toProcess = mutableMapOf<PostEntity, List<ImageEntity>>()
+
+            for ((post, images) in posts) {
+                if (images.isEmpty()) {
+                    continue
+                }
+                if (isPending(post.postId)) {
+                    continue
+                }
+                toProcess[post] = images
+            }
+
+            toProcess.forEach { (post, images) ->
+                post.status = Status.PENDING
+                images.forEach { image ->
+                    with(image) {
+                        this.status = Status.PENDING
+                        this.downloaded = 0
+                    }
+                }
+            }
+
+            transaction {
+                dataTransaction.updatePosts(
+                    toProcess.keys.toList()
+                )
+                dataTransaction.updateImages(
+                    toProcess.values.flatten()
+                )
+            }
+
+
+            toProcess.entries.forEach { (post, images) ->
+                images.forEach { image ->
+                    log.debug("Enqueuing a job for ${image.url}")
+                    val imageDownloadRunnable = ImageDownloadRunnable(
+                        image, post.rank, settingsService.settings
+                    )
+                    pending.computeIfAbsent(
+                        image.host
+                    ) { mutableListOf() }
+                    pending[image.host]!!.add(imageDownloadRunnable)
+                }
+            }
+
+            condition.signal()
+        }
+    }
+
+    private fun isPending(postId: Long): Boolean {
+        return pending.values.flatten().any { it.imageEntity.postId == postId }
+    }
+
+    private fun isRunning(postId: Long): Boolean {
+        return running.values.flatten().any { it.imageEntity.postId == postId }
+    }
+
+    private fun stopAll() {
+        lock.withLock {
+            pending.values.clear()
+            running.values.flatten().forEach { obj: ImageDownloadRunnable -> obj.stop() }
+            while (running.values.flatten().count { !it.completed } > 0) {
+                Thread.sleep(100)
+            }
+            dataTransaction.findAllNonCompletedPostIds().forEach {
+                dataTransaction.stopImagesByPostIdAndIsNotCompleted(it)
+                dataTransaction.finishPost(it)
+            }
+        }
+    }
+
+    private fun stopInternal(postIds: List<Long>) {
+        lock.withLock {
+            for (postId in postIds) {
+                pending.values.forEach { pending ->
+                    pending.removeIf { it.imageEntity.postId == postId }
+                }
+                running.values.flatten()
+                    .filter { p: ImageDownloadRunnable -> p.imageEntity.postId == postId }
+                    .forEach { obj: ImageDownloadRunnable -> obj.stop() }
+                while (running.values.flatten()
+                        .count { !it.completed && it.imageEntity.postId == postId } > 0
+                ) {
+                    Thread.sleep(100)
+                }
+            }
+            postIds.forEach {
+                dataTransaction.stopImagesByPostIdAndIsNotCompleted(it)
+                dataTransaction.finishPost(it)
+            }
+        }
+    }
+
+    private fun canRun(host: Byte): Boolean {
+        val totalRunning = running.values.sumOf { it.size }
+        return (running[host]!!.size < settingsService.settings.connectionSettings.maxConcurrentPerHost && if (settingsService.settings.connectionSettings.maxGlobalConcurrent == 0) totalRunning < maxPoolSize else totalRunning < settingsService.settings.connectionSettings.maxGlobalConcurrent)
+    }
+
+    private fun candidateCount(): Map<Byte, Int> {
+        val map: MutableMap<Byte, Int> = mutableMapOf()
+        Host.Companion.getHosts().values.forEach { host: Byte ->
+            val imageDownloadRunnableList: List<ImageDownloadRunnable> = running.computeIfAbsent(
+                host
+            ) { mutableListOf() }
+            val count: Int =
+                settingsService.settings.connectionSettings.maxConcurrentPerHost - imageDownloadRunnableList.size
+            log.debug("Download slots for $host: $count")
+            map[host] = count
+        }
+        return map
+    }
+
+    private fun getCandidates(candidateCount: Map<Byte, Int>): List<ImageDownloadRunnable> {
+        val hostIntegerMap: MutableMap<Byte, Int> = candidateCount.toMutableMap()
+        val candidates: MutableList<ImageDownloadRunnable> = mutableListOf()
+        hosts@ for (host in pending.keys) {
+
+            val list: List<ImageDownloadRunnable> =
+                pending[host]!!.sortedWith(Comparator.comparingInt<ImageDownloadRunnable> { it.postRank }
+                    .thenComparingInt { it.imageEntity.index })
+
+            for (imageDownloadRunnable in list) {
+                val count = hostIntegerMap[host] ?: 0
+                if (count > 0) {
+                    candidates.add(imageDownloadRunnable)
+                    hostIntegerMap[host] = count - 1
+                } else {
+                    continue@hosts
+                }
+            }
+        }
+        return candidates
+    }
+
+    private fun scheduleForDownload(imageDownloadRunnable: ImageDownloadRunnable) {
+        log.debug("Scheduling a job for ${imageDownloadRunnable.imageEntity.url}")
+        eventBus.publishEvent(QueueStateEvent(QueueState(runningCount(), pendingCount())))
+        Failsafe.with<Any, RetryPolicy<Any>>(retryPolicyService.buildRetryPolicy("Failed to download ${imageDownloadRunnable.imageEntity.url}: "))
+            .with(downloadRunner)
+            .onFailure {
+                log.error(
+                    "Failed to download ${imageDownloadRunnable.imageEntity.url} after ${it.attemptCount} tries",
+                    it.exception
+                )
+                val image = imageDownloadRunnable.imageEntity
+                image.status = Status.ERROR
+                dataTransaction.updateImage(image)
+            }
+            .onComplete {
+                afterJobFinish(imageDownloadRunnable)
+                eventBus.publishEvent(
+                    QueueStateEvent(
+                        QueueState(
+                            runningCount(), pendingCount()
+                        )
+                    )
+                )
+                eventBus.publishEvent(ErrorCountEvent(ErrorCount(dataTransaction.countImagesInError())))
+                log.debug(
+                    "Finished downloading ${imageDownloadRunnable.imageEntity.url}"
+                )
+            }.runAsync(imageDownloadRunnable)
+    }
+
+    private fun afterJobFinish(imageDownloadRunnable: ImageDownloadRunnable) {
+        lock.withLock {
+            val image = imageDownloadRunnable.imageEntity
+            running[image.host]!!.remove(imageDownloadRunnable)
+            if (!isPending(image.postId) && !isRunning(
+                    image.postId
+                ) && !imageDownloadRunnable.stopped
+            ) {
+                dataTransaction.finishPost(image.postId, true)
+            }
+            condition.signal()
+        }
+    }
+
+    private fun pendingCount(): Int {
+        return pending.values.sumOf { it.size }
+    }
+
+    private fun runningCount(): Int {
+        return running.values.sumOf { it.size }
+    }
+}

vripper-core/src/main/kotlin/me/vripper/services/DownloadSpeedService.kt

@@ -8,7 +8,7 @@ import me.vripper.event.QueueStateEvent
 import me.vripper.model.DownloadSpeed
 import java.util.concurrent.atomic.AtomicLong
 
-class DownloadSpeedService(
+internal class DownloadSpeedService(
     private val eventBus: EventBus,
 ) {
 
@@ -19,9 +19,11 @@ class DownloadSpeedService(
     private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
     private val bytesCount = AtomicLong(0)
     private var job: Job? = null
+    private var queueStateUpdateJob: Job? = null
 
-    init {
-        coroutineScope.launch {
+    fun init() {
+        queueStateUpdateJob?.cancel()
+        queueStateUpdateJob = coroutineScope.launch {
             eventBus.events.filterIsInstance(QueueStateEvent::class).collect {
                 if (it.queueState.running + it.queueState.remaining > 0) {
                     if (job == null || job?.isActive == false) {
@@ -30,7 +32,6 @@ class DownloadSpeedService(
                             while (isActive) {
                                 delay(DOWNLOAD_POLL_RATE.toLong())
                                 val newValue = bytesCount.getAndSet(0)
-
                                 eventBus.publishEvent(DownloadSpeedEvent(DownloadSpeed(((newValue * 1000) / DOWNLOAD_POLL_RATE))))
                             }
                         }
@@ -46,6 +47,11 @@ class DownloadSpeedService(
         }
     }
 
+    fun halt() {
+        queueStateUpdateJob?.cancel()
+        job?.cancel()
+    }
+
     fun reportDownloadedBytes(count: Long) {
         bytesCount.addAndGet(count)
     }

vripper-core/src/main/kotlin/me/vripper/services/HTTPService.kt

@@ -1,100 +1,69 @@
 package me.vripper.services
 
 import kotlinx.coroutines.*
-import kotlinx.coroutines.flow.filterIsInstance
-import me.vripper.event.EventBus
-import me.vripper.event.SettingsUpdateEvent
+import me.vripper.utilities.ApplicationProperties
 import org.apache.hc.client5.http.config.ConnectionConfig
 import org.apache.hc.client5.http.config.RequestConfig
 import org.apache.hc.client5.http.cookie.StandardCookieSpec
 import org.apache.hc.client5.http.impl.DefaultRedirectStrategy
 import org.apache.hc.client5.http.impl.classic.CloseableHttpClient
 import org.apache.hc.client5.http.impl.classic.HttpClients
-import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager
+import org.apache.hc.client5.http.impl.io.BasicHttpClientConnectionManager
 import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder
+import org.apache.hc.client5.http.io.HttpClientConnectionManager
 import org.apache.hc.core5.pool.PoolConcurrencyPolicy
 import org.apache.hc.core5.pool.PoolReusePolicy
-import org.apache.hc.core5.util.TimeValue
 import org.apache.hc.core5.util.Timeout
 
-class HTTPService(
-    private val eventBus: EventBus,
-    settingsService: SettingsService
-) {
-
-    companion object {
-        const val USER_AGENT =
-            "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/118.0"
-    }
-
+internal class HTTPService {
     private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
 
-    private lateinit var pcm: PoolingHttpClientConnectionManager
-    private lateinit var rc: RequestConfig
-    private lateinit var cc: ConnectionConfig
-    lateinit var client: CloseableHttpClient
-    private var connectionTimeout = settingsService.settings.connectionSettings.timeout
+    private var pcm: HttpClientConnectionManager = BasicHttpClientConnectionManager()
+    private var rc: RequestConfig = RequestConfig.DEFAULT
+    private var cc: ConnectionConfig = ConnectionConfig.DEFAULT
+    private var connectionExpiryJob: Job? = null
+    var client: CloseableHttpClient = HttpClients.createDefault()
 
-    init {
-        buildRequestConfig()
-        buildConnectionConfig()
-        buildConnectionPool()
-        buildClientBuilder()
-        coroutineScope.launch {
-            pcm.closeIdle(TimeValue.ofSeconds(60))
-            delay(15000)
-        }
-        coroutineScope.launch {
-            eventBus
-                .events
-                .filterIsInstance(SettingsUpdateEvent::class)
-                .collect {
-                    if (connectionTimeout != it.settings.connectionSettings.timeout) {
-                        connectionTimeout = it.settings.connectionSettings.timeout
-                        client.close()
-                        pcm.close()
-                        buildRequestConfig()
-                        buildConnectionConfig()
-                        buildConnectionPool()
-                        buildClientBuilder()
-                    }
-                }
-        }
-    }
-
-    private fun buildConnectionPool() {
+    fun buildConnectionPool() {
+        pcm.close()
+        connectionExpiryJob?.cancel()
         pcm = PoolingHttpClientConnectionManagerBuilder.create()
-            .setPoolConcurrencyPolicy(PoolConcurrencyPolicy.STRICT)
-            .setConnPoolPolicy(PoolReusePolicy.LIFO)
+            .setPoolConcurrencyPolicy(PoolConcurrencyPolicy.LAX)
+            .setConnPoolPolicy(PoolReusePolicy.FIFO)
             .setDefaultConnectionConfig(cc)
             .setMaxConnTotal(Int.MAX_VALUE)
             .setMaxConnPerRoute(Int.MAX_VALUE)
-            .build()
+            .build().also {
+                connectionExpiryJob = coroutineScope.launch {
+                    while (isActive) {
+                        it.closeExpired()
+                        delay(60_000)
+                    }
+                }
+            }
     }
 
-    private fun buildRequestConfig() {
+    fun buildRequestConfig(connectionTimeout: Long) {
         rc = RequestConfig.custom()
-            .setConnectionRequestTimeout(Timeout.ofSeconds(connectionTimeout.toLong()))
+            .setConnectionRequestTimeout(Timeout.ofSeconds(connectionTimeout))
             .setCookieSpec(StandardCookieSpec.RELAXED)
             .build()
     }
 
-    private fun buildConnectionConfig() {
+    fun buildConnectionConfig(connectionTimeout: Long) {
         cc = ConnectionConfig.custom()
-            .setConnectTimeout(Timeout.ofSeconds(connectionTimeout.toLong()))
-            .setSocketTimeout(Timeout.ofSeconds(connectionTimeout.toLong()))
-            .setTimeToLive(TimeValue.ofMinutes(10))
+            .setConnectTimeout(Timeout.ofSeconds(connectionTimeout))
+            .setSocketTimeout(Timeout.ofSeconds(connectionTimeout))
             .build()
     }
 
-    private fun buildClientBuilder() {
+    fun buildClientBuilder() {
         client = HttpClients.custom()
             .setConnectionManager(pcm)
             .setRedirectStrategy(DefaultRedirectStrategy.INSTANCE)
-            .setUserAgent(USER_AGENT)
+            .setUserAgent(ApplicationProperties.USER_AGENT)
             .disableAutomaticRetries()
             .setDefaultRequestConfig(rc)
             .build()
-
     }
 }

vripper-core/src/main/kotlin/me/vripper/services/IAppEndpointService.kt

@@ -17,13 +17,10 @@ interface IAppEndpointService {
     fun onDeletePosts(): Flow<Long>
     fun onUpdateMetadata(): Flow<Metadata>
     suspend fun findImagesByPostId(postId: Long): List<Image>
-    fun onUpdateImages(postId: Long): Flow<Image>
+    fun onUpdateImagesByPostId(postId: Long): Flow<Image>
+    fun onUpdateImages(): Flow<Image>
     fun onStopped(): Flow<Long>
-    suspend fun logClear()
-    suspend fun findAllLogs(): List<LogEntry>
     fun onNewLog(): Flow<LogEntry>
-    fun onUpdateLog(): Flow<LogEntry>
-    fun onDeleteLogs(): Flow<Long>
     fun onNewThread(): Flow<Thread>
     fun onUpdateThread(): Flow<Thread>
     fun onDeleteThread(): Flow<Long>
@@ -33,10 +30,10 @@ interface IAppEndpointService {
     suspend fun threadClear()
     suspend fun grab(threadId: Long): List<PostSelection>
     suspend fun download(posts: List<ThreadPostId>)
-    fun onDownloadSpeed(): Flow<Long>
+    fun onDownloadSpeed(): Flow<DownloadSpeed>
     fun onVGUserUpdate(): Flow<String>
     fun onQueueStateUpdate(): Flow<QueueState>
-    fun onErrorCountUpdate(): Flow<Int>
+    fun onErrorCountUpdate(): Flow<ErrorCount>
     fun onTasksRunning(): Flow<Boolean>
     suspend fun getSettings(): Settings
     suspend fun saveSettings(settings: Settings)
@@ -45,4 +42,7 @@ interface IAppEndpointService {
     suspend fun loggedInUser(): String
     suspend fun getVersion(): String
     suspend fun renameToFirst(postIds: List<Long>)
+    suspend fun dbMigration(): String
+    suspend fun initLogger()
+    fun connectionState(): String
 }

vripper-core/src/main/kotlin/me/vripper/services/LogService.kt

@@ -0,0 +1,49 @@
+package me.vripper.services
+
+import kotlinx.coroutines.*
+import me.vripper.RingAppender
+import me.vripper.entities.LogEntryEntity
+import me.vripper.event.EventBus
+import me.vripper.event.LogCreateEvent
+import me.vripper.utilities.StackTraceElementUtils
+import java.time.ZoneId
+import java.time.format.DateTimeFormatter
+
+class LogService(private val eventBus: EventBus) {
+    private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
+    private val dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")
+    private var job: Job? = null
+
+    fun init() {
+        if (job != null) {
+            return
+        }
+        job = coroutineScope.launch {
+            while (isActive) {
+                while (!RingAppender.Static.events.isEmpty()) {
+                    RingAppender.Static.events.poll()?.also { loggingEvent ->
+                        eventBus.publishEvent(
+                            LogCreateEvent(
+                                LogEntryEntity(
+                                    loggingEvent.sequenceNumber,
+                                    loggingEvent.instant.atZone(ZoneId.systemDefault()).toLocalDateTime()
+                                        .format(dateTimeFormatter),
+                                    loggingEvent.threadName,
+                                    loggingEvent.loggerName,
+                                    loggingEvent.level.levelStr,
+                                    loggingEvent.formattedMessage,
+                                    loggingEvent.throwableProxy?.let { throwableProxy ->
+                                        StackTraceElementUtils.format(
+                                            throwableProxy
+                                        )
+                                    } ?: ""
+                                )
+                            )
+                        )
+                    }
+                }
+                delay(777)
+            }
+        }
+    }
+}

vripper-core/src/main/kotlin/me/vripper/services/MetadataService.kt

@@ -1,113 +1,29 @@
 package me.vripper.services
 
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.SupervisorJob
-import kotlinx.coroutines.launch
-import kotlinx.coroutines.sync.withPermit
-import me.vripper.entities.MetadataEntity
-import me.vripper.exception.DownloadException
-import me.vripper.exception.VripperException
-import me.vripper.utilities.HtmlUtils
-import me.vripper.utilities.RequestLimit
-import me.vripper.utilities.Tasks
-import me.vripper.utilities.XpathUtils
-import org.apache.hc.client5.http.classic.methods.HttpGet
-import org.apache.hc.core5.net.URIBuilder
-import org.w3c.dom.Node
-import java.util.concurrent.atomic.AtomicBoolean
-import java.util.stream.Collectors
+import me.vripper.tasks.FetchMetadataTask
+import me.vripper.utilities.taskRunner
 
-class MetadataService(
-    private val httpService: HTTPService,
-    private val settingsService: SettingsService,
+internal class MetadataService(
     private val dataTransaction: DataTransaction,
-    private val vgAuthService: VGAuthService,
+    private val settingsService: SettingsService,
 ) {
 
-    private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
-    private val dictionary: List<String> = mutableListOf("download", "link", "rapidgator", "filefactory", "filefox")
-
-    fun init() {
+    fun fetchExisting() {
+        if (!settingsService.settings.viperSettings.fetchMetadata) {
+            return
+        }
         dataTransaction.findAllPosts().filter { dataTransaction.findMetadataByPostId(it.postId).isEmpty }
             .map { it.postId }.forEach(::fetchMetadata)
     }
 
     fun fetchMetadata(postId: Long) {
-        Tasks.increment()
-        coroutineScope.launch {
-            RequestLimit.semaphore.withPermit {
-                try {
-                    val httpGet = HttpGet(URIBuilder(settingsService.settings.viperSettings.host + "/threads/").also {
-                        it.setParameter(
-                            "p", postId.toString()
-                        )
-                    }.build())
-
-                    val metadataEntity = httpService.client.execute(httpGet, vgAuthService.context) {
-                        if (it.code / 100 != 2) {
-                            throw DownloadException("Unexpected response code '${it.code}' for $httpGet")
-                        }
-                        val document = HtmlUtils.clean(it.entity.content)
-                        val postNode: Node = XpathUtils.getAsNode(
-                            document,
-                            "//li[@id='post_$postId']/div[contains(@class, 'postdetails')]",
-
-                            ) ?: throw VripperException("Unable to find post #'$postId'")
-
-                        val postedBy: String = XpathUtils.getAsNode(
-                            postNode, "./div[contains(@class, 'userinfo')]//a[contains(@class, 'username')]//font"
-                        )?.textContent?.trim()
-                            ?: throw VripperException("Unable to find the poster for post #'$postId'")
-
-
-                        val node: Node = XpathUtils.getAsNode(
-                            document, java.lang.String.format("//div[@id='post_message_%s']", postId)
-                        ) ?: throw VripperException("Unable to locate post content")
-                        val titles = findTitleInContent(node)
-                        MetadataEntity(postId, MetadataEntity.Data(postedBy, titles))
-                    }
-                    try {
-                        dataTransaction.saveMetadata(metadataEntity)
-                    } catch (e: Exception) {
-                        e.printStackTrace()
-                    }
-                } finally {
-                    Tasks.decrement()
-                }
-            }
-        }
-    }
-
-    private fun findTitleInContent(node: Node): List<String> {
-        val altTitle: MutableList<String> = mutableListOf()
-        findTitle(node, altTitle, AtomicBoolean(true))
-        return altTitle.stream().distinct().collect(Collectors.toList())
-    }
-
-    private fun findTitle(node: Node, altTitle: MutableList<String>, keepGoing: AtomicBoolean) {
-        if (!keepGoing.get()) {
+        if (!settingsService.settings.viperSettings.fetchMetadata) {
             return
         }
-        if (node.nodeName == "a" || node.nodeName == "img") {
-            keepGoing.set(false)
-            return
-        }
-        if (node.nodeType == Node.ELEMENT_NODE) {
-            for (i in 0 until node.childNodes.length) {
-                val item = node.childNodes.item(i)
-                findTitle(item, altTitle, keepGoing)
-                if (!keepGoing.get()) {
-                    return
-                }
-            }
-        } else if (node.nodeType == Node.TEXT_NODE) {
-            val text = node.textContent.trim()
-            if (text.isNotBlank() && dictionary.stream().noneMatch { e ->
-                    text.lowercase().contains(e.lowercase())
-                }) {
-                altTitle.add(text)
-            }
-        }
+        taskRunner.submit(
+            FetchMetadataTask(
+                postId
+            )
+        )
     }
 }

vripper-core/src/main/kotlin/me/vripper/services/RetryPolicyService.kt

@@ -1,43 +1,17 @@
 package me.vripper.services
 
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.SupervisorJob
-import kotlinx.coroutines.flow.filterIsInstance
-import kotlinx.coroutines.launch
-import me.vripper.event.EventBus
-import me.vripper.event.SettingsUpdateEvent
-import net.jodah.failsafe.RetryPolicy
-import net.jodah.failsafe.event.ExecutionAttemptedEvent
+import dev.failsafe.RetryPolicy
+import me.vripper.utilities.LoggerDelegate
 import java.time.temporal.ChronoUnit
 
-class RetryPolicyService(
-    val eventBus: EventBus, settingsService: SettingsService
-) {
-    private val log by me.vripper.delegate.LoggerDelegate()
-    private var maxAttempts: Int = settingsService.settings.connectionSettings.maxAttempts
-    private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
+internal class RetryPolicyService {
+    private val log by LoggerDelegate()
+    var maxAttempts: Int = 3
 
-    init {
-        coroutineScope.launch {
-            eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collect {
-                if (maxAttempts != it.settings.connectionSettings.maxAttempts) {
-                    maxAttempts = it.settings.connectionSettings.maxAttempts
-                }
-            }
-        }
-    }
-
-    fun <T> buildRetryPolicyForDownload(): RetryPolicy<T> {
-        return RetryPolicy<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts).onFailedAttempt {
-                log.warn("#${it.attemptCount} tries failed", it.lastFailure)
-            }
-    }
-
-    fun <T> buildGenericRetryPolicy(): RetryPolicy<T> {
-        return RetryPolicy<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts)
-            .onFailedAttempt { e: ExecutionAttemptedEvent<T> ->
-                log.warn("#${e.attemptCount} tries failed", e.lastFailure)
-            }
+    fun <T> buildRetryPolicy(message: String): RetryPolicy<T> {
+        return RetryPolicy.builder<T>().withDelay(2, 5, ChronoUnit.SECONDS).withMaxAttempts(maxAttempts)
+            .onFailedAttempt {
+                log.warn(message + "#${it.attemptCount} tries failed", it.lastException)
+            }.build()
     }
 }

vripper-core/src/main/kotlin/me/vripper/services/SettingsService.kt

@@ -1,9 +1,6 @@
 package me.vripper.services
 
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.SupervisorJob
-import kotlinx.coroutines.launch
+import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
 import kotlinx.serialization.json.decodeFromStream
@@ -12,18 +9,18 @@ import me.vripper.event.SettingsUpdateEvent
 import me.vripper.exception.ValidationException
 import me.vripper.model.Settings
 import me.vripper.utilities.ApplicationProperties.VRIPPER_DIR
-import org.apache.commons.codec.digest.DigestUtils
+import me.vripper.utilities.LoggerDelegate
+import me.vripper.utilities.md5Hex
 import java.io.FileWriter
 import java.nio.file.*
 import kotlin.io.path.readText
 
 class SettingsService(private val eventBus: EventBus) {
 
-    private val log by me.vripper.delegate.LoggerDelegate()
+    private val log by LoggerDelegate()
     private val configPath = VRIPPER_DIR.resolve("config.json")
     private val customProxiesPath = VRIPPER_DIR.resolve("proxies.json")
     private val proxies: MutableSet<String> = HashSet()
-    private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
 
     private val json = Json {
         encodeDefaults = true
@@ -32,19 +29,13 @@ class SettingsService(private val eventBus: EventBus) {
     }
     var settings = Settings()
 
-    init {
-        init()
-    }
-
-    private fun init() {
+    fun init() {
         loadViperProxies()
         restore()
-        coroutineScope.launch {
-            eventBus.publishEvent(SettingsUpdateEvent(settings))
-        }
-
+        eventBus.publishEvent(SettingsUpdateEvent(settings))
     }
 
+    @OptIn(ExperimentalSerializationApi::class)
     private fun loadViperProxies() {
         try {
             SettingsService::class.java.getResourceAsStream("/proxies.json")?.use {
@@ -57,7 +48,7 @@ class SettingsService(private val eventBus: EventBus) {
                             customProxiesPath.readText()
                         )
                     } catch (e: Exception) {
-                        e.printStackTrace()
+                        log.error("Failed to read custom proxies", e)
                         emptyList()
                     }
                 } else {
@@ -91,7 +82,7 @@ class SettingsService(private val eventBus: EventBus) {
         check(settings)
         val viperSettings = if (settings.viperSettings.login) {
             if (this.settings.viperSettings.password != settings.viperSettings.password) {
-                settings.viperSettings.copy(password = DigestUtils.md5Hex(settings.viperSettings.password))
+                settings.viperSettings.copy(password = md5Hex(settings.viperSettings.password))
             } else {
                 settings.viperSettings
             }
@@ -100,9 +91,7 @@ class SettingsService(private val eventBus: EventBus) {
         }
         this.settings = settings.copy(viperSettings = viperSettings)
         save()
-        coroutineScope.launch {
-            eventBus.publishEvent(SettingsUpdateEvent(this@SettingsService.settings))
-        }
+        eventBus.publishEvent(SettingsUpdateEvent(this@SettingsService.settings))
     }
 
     private fun restore() {
@@ -111,7 +100,7 @@ class SettingsService(private val eventBus: EventBus) {
                 settings = json.decodeFromString(configPath.readText())
             }
         } catch (e: Exception) {
-            log.error("Failed restore user settings", e)
+            log.error("Failed to restore user settings", e)
             settings = Settings()
         }
         if (!proxies.contains(settings.viperSettings.host)) {
@@ -146,7 +135,7 @@ class SettingsService(private val eventBus: EventBus) {
     fun check(settings: Settings) {
         val path: Path = try {
             Paths.get(settings.downloadSettings.downloadPath)
-        } catch (e: InvalidPathException) {
+        } catch (_: InvalidPathException) {
             throw ValidationException(
                 String.format(
                     "%s is invalid", settings.downloadSettings.downloadPath
@@ -197,5 +186,11 @@ class SettingsService(private val eventBus: EventBus) {
                 "Invalid clipboard monitoring polling rate settings, values must be >= 500"
             )
         }
+
+        if (settings.viperSettings.requestLimit < 1 || settings.viperSettings.requestLimit > 5) {
+            throw ValidationException(
+                "Invalid request rate limit, values must be in [1,5]"
+            )
+        }
     }
 }

vripper-core/src/main/kotlin/me/vripper/services/ThreadCacheService.kt

@@ -2,39 +2,28 @@ package me.vripper.services
 
 import com.github.benmanes.caffeine.cache.Caffeine
 import com.github.benmanes.caffeine.cache.LoadingCache
-import kotlinx.coroutines.*
-import kotlinx.coroutines.flow.filterIsInstance
-import kotlinx.coroutines.sync.withPermit
-import me.vripper.event.EventBus
-import me.vripper.event.SettingsUpdateEvent
-import me.vripper.parser.ThreadItem
-import me.vripper.parser.ThreadLookupAPIParser
-import me.vripper.utilities.RequestLimit
-import java.util.*
+import me.vripper.vgapi.ThreadItem
+import me.vripper.vgapi.ThreadLookupAPIParser
 import java.util.concurrent.ExecutionException
 import java.util.concurrent.TimeUnit
 
-class ThreadCacheService(val eventBus: EventBus) {
-
-    private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
-
-    fun init() {
-        coroutineScope.launch {
-            eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collect {
-                cache.invalidateAll()
-            }
-        }
-    }
+internal class ThreadCacheService(val dataTransaction: DataTransaction) {
 
     private val cache: LoadingCache<Long, ThreadItem> =
         Caffeine.newBuilder().expireAfterWrite(20, TimeUnit.MINUTES).build { threadId ->
-            runBlocking {
-                RequestLimit.semaphore.withPermit {
-                    ThreadLookupAPIParser(threadId).parse()
+            val threadItem = ThreadLookupAPIParser(threadId).parse()
+            dataTransaction.findThreadByThreadId(threadItem.threadId).ifPresent {
+                if (threadItem.postItemList.isNotEmpty()) {
+                    dataTransaction.update(it.copy(total = threadItem.postItemList.size))
                 }
             }
+            threadItem
         }
 
+    fun invalidate() {
+        cache.invalidateAll()
+    }
+
     @Throws(ExecutionException::class)
     operator fun get(threadId: Long): ThreadItem {
         return cache[threadId]

vripper-core/src/main/kotlin/me/vripper/services/VGAuthService.kt

@@ -1,72 +1,52 @@
 package me.vripper.services
 
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.SupervisorJob
-import kotlinx.coroutines.flow.filterIsInstance
-import kotlinx.coroutines.launch
 import me.vripper.entities.PostEntity
 import me.vripper.event.EventBus
-import me.vripper.event.SettingsUpdateEvent
 import me.vripper.event.VGUserLoginEvent
 import me.vripper.exception.VripperException
 import me.vripper.model.Settings
-import me.vripper.tasks.LeaveThanksRunnable
-import me.vripper.utilities.GLOBAL_EXECUTOR
+import me.vripper.tasks.LeaveThanksTask
+import me.vripper.utilities.LoggerDelegate
+import me.vripper.utilities.taskRunner
 import org.apache.hc.client5.http.classic.methods.HttpPost
 import org.apache.hc.client5.http.cookie.BasicCookieStore
 import org.apache.hc.client5.http.cookie.Cookie
 import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
+import org.apache.hc.client5.http.impl.cookie.BasicClientCookie
 import org.apache.hc.client5.http.protocol.HttpClientContext
 import org.apache.hc.core5.http.io.entity.EntityUtils
 import org.apache.hc.core5.http.message.BasicNameValuePair
-import java.util.concurrent.CompletableFuture
 
-class VGAuthService(
+internal class VGAuthService(
     private val cm: HTTPService,
-    private val settingsService: SettingsService,
-    private val eventBus: EventBus
+    private val eventBus: EventBus,
 ) {
-    private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
-    private val log by me.vripper.delegate.LoggerDelegate()
-    val context: HttpClientContext = HttpClientContext.create()
+    private val log by LoggerDelegate()
+    private val vgCookies: MutableList<Cookie> = mutableListOf()
     var loggedUser = ""
-
     private var authenticated = false
 
-    init {
-        context.cookieStore = BasicCookieStore()
-        coroutineScope.launch {
-            eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collect {
-                authenticate(it.settings)
-            }
-        }
-        authenticate(settingsService.settings)
-    }
-
-    private fun authenticate(settings: Settings) {
-        authenticated = false
+    fun authenticate(settings: Settings) {
         if (!settings.viperSettings.login) {
             log.debug("Authentication option is disabled")
-            context.cookieStore.clear()
+            authenticated = false
             loggedUser = ""
-            coroutineScope.launch {
-                eventBus.publishEvent(VGUserLoginEvent(loggedUser))
+            synchronized(vgCookies) {
+                vgCookies.clear()
             }
-
+            eventBus.publishEvent(VGUserLoginEvent(loggedUser))
             return
         }
         val username = settings.viperSettings.username
         val password = settings.viperSettings.password
         if (username.isEmpty() || password.isEmpty()) {
             log.error("Cannot authenticate with ViperGirls credentials, username or password is empty")
-            context.cookieStore.clear()
+            authenticated = false
             loggedUser = ""
-
-            coroutineScope.launch {
-                eventBus.publishEvent(VGUserLoginEvent(loggedUser))
+            synchronized(vgCookies) {
+                vgCookies.clear()
             }
-
+            eventBus.publishEvent(VGUserLoginEvent(loggedUser))
             return
         }
         val postAuth = HttpPost(settings.viperSettings.host + "/login.php?do=login").also {
@@ -78,13 +58,13 @@ class VGAuthService(
                     BasicNameValuePair("vb_login_md5password", password)
                 )
             )
-            it.addHeader("Referer", settings.viperSettings.host)
-            it.addHeader(
-                "Host", settings.viperSettings.host.replace("https://", "").replace("http://", "")
-            )
         }
-
+        log.info("Authenticating: ${postAuth.uri}")
         try {
+            val context = HttpClientContext.create().apply {
+                cookieStore =
+                    BasicCookieStore()
+            }
             cm.client.execute(postAuth, context) { response ->
                 if (response.code / 100 != 2) {
                     throw VripperException("Unexpected response code returned ${response.code}")
@@ -92,38 +72,72 @@ class VGAuthService(
                 val responseBody = EntityUtils.toString(response.entity)
                 log.debug("Authentication with ViperGirls response body:{}", responseBody)
             }
-            if (context.cookieStore.cookies.stream().map { obj: Cookie -> obj.name }
-                    .noneMatch { e: String -> e == "vg_userid" }) {
+
+            val userIdCookie = context.cookieStore.cookies.find { it.name == "vg_userid" }
+            val passwordCookie = context.cookieStore.cookies.find { it.name == "vg_password" }
+
+            if (userIdCookie == null || passwordCookie == null) {
                 log.error(
-                    "Failed to authenticate user with {}, missing vg_userid cookie",
+                    "Failed to authenticate user with {}, missing vg_userid/vg_password cookie",
                     settings.viperSettings.host
                 )
+                eventBus.publishEvent(VGUserLoginEvent(loggedUser))
                 return
             }
-        } catch (e: Exception) {
-            context.cookieStore.clear()
-            loggedUser = ""
-
-            coroutineScope.launch {
-                eventBus.publishEvent(VGUserLoginEvent(loggedUser))
+            synchronized(vgCookies) {
+                vgCookies.clear()
+                vgCookies.add(BasicClientCookie(userIdCookie.name, userIdCookie.value).apply {
+                    domain = userIdCookie.domain
+                })
+                vgCookies.add(BasicClientCookie(passwordCookie.name, passwordCookie.value).apply {
+                    domain = passwordCookie.domain
+                })
             }
-
+        } catch (e: Exception) {
             log.error(
                 "Failed to authenticate user with " + settings.viperSettings.host, e
             )
+            authenticated = false
+            loggedUser = ""
+            eventBus.publishEvent(VGUserLoginEvent(loggedUser))
             return
         }
         authenticated = true
         loggedUser = username
-
-        coroutineScope.launch {
-            eventBus.publishEvent(VGUserLoginEvent(loggedUser))
-        }
+        eventBus.publishEvent(VGUserLoginEvent(loggedUser))
+        log.info("Successfully logged in as: $loggedUser")
     }
 
     fun leaveThanks(postEntity: PostEntity) {
-        CompletableFuture.runAsync(
-            LeaveThanksRunnable(postEntity, authenticated, context), GLOBAL_EXECUTOR
+        val context = createVgContext()
+        taskRunner.submit(
+            LeaveThanksTask(postEntity, authenticated, context)
         )
     }
+
+    fun createVgContext(): HttpClientContext {
+        val context = HttpClientContext().apply { cookieStore = BasicCookieStore() }
+        synchronized(vgCookies) {
+            if (authenticated && vgCookies.isNotEmpty()) {
+                vgCookies.forEach { c -> context.cookieStore.addCookie(c) }
+            }
+        }
+        return context
+    }
+
+    fun createClickContext(): HttpClientContext {
+        val context = HttpClientContext().apply { cookieStore = BasicCookieStore() }
+        synchronized(vgCookies) {
+            if (authenticated && vgCookies.isNotEmpty()) {
+                vgCookies.forEach { c ->
+                    context.cookieStore.addCookie(
+                        BasicClientCookie(
+                            c.name,
+                            c.value
+                        ).apply { domain = "viper.click" })
+                }
+            }
+        }
+        return context
+    }
 }

vripper-core/src/main/kotlin/me/vripper/tables/LogTable.kt

@@ -1,11 +0,0 @@
-package me.vripper.tables
-
-import org.jetbrains.exposed.dao.id.LongIdTable
-import org.jetbrains.exposed.sql.javatime.datetime
-
-object LogTable : LongIdTable(name = "LOG", columnName = "ID") {
-    val type = varchar("TYPE", 8)
-    val status = varchar("STATUS", 10)
-    val time = datetime("TIME")
-    val message = varchar("MESSAGE", 500)
-}

vripper-core/src/main/kotlin/me/vripper/tasks/AddPostRunnable.kt

@@ -1,112 +0,0 @@
-package me.vripper.tasks
-
-import kotlinx.coroutines.runBlocking
-import kotlinx.coroutines.sync.withPermit
-import me.vripper.download.DownloadService
-import me.vripper.entities.LogEntryEntity
-import me.vripper.model.ThreadPostId
-import me.vripper.parser.PostItem
-import me.vripper.parser.PostLookupAPIParser
-import me.vripper.services.DataTransaction
-import me.vripper.services.MetadataService
-import me.vripper.services.SettingsService
-import me.vripper.services.ThreadCacheService
-import me.vripper.utilities.RequestLimit
-import me.vripper.utilities.Tasks
-import org.koin.core.component.KoinComponent
-import org.koin.core.component.inject
-
-class AddPostRunnable(private val items: List<ThreadPostId>) : KoinComponent, Runnable {
-    private val log by me.vripper.delegate.LoggerDelegate()
-    private val dataTransaction: DataTransaction by inject()
-    private val settingsService: SettingsService by inject()
-    private val downloadService: DownloadService by inject()
-    private val threadCacheService: ThreadCacheService by inject()
-    private val metadataService: MetadataService by inject()
-
-    override fun run() {
-        try {
-            Tasks.increment()
-            val toProcess = mutableListOf<PostItem>()
-            for ((threadId, postId) in items) {
-                if (dataTransaction.exists(postId)) {
-                    continue
-                }
-
-                val link =
-                    "https://${settingsService.settings.viperSettings.host}/threads/$threadId?p=$postId&viewfull=1#post$postId"
-
-                val cachedThread = threadCacheService.getIfPresent(threadId)
-                val threadItem = cachedThread ?: runBlocking {
-                    RequestLimit.semaphore.withPermit {
-                        PostLookupAPIParser(
-                            threadId, postId
-                        ).parse()
-                    }
-                }
-
-                if (threadItem == null) {
-                    dataTransaction.saveLog(
-                        LogEntryEntity(
-                            type = LogEntryEntity.Type.POST,
-                            status = LogEntryEntity.Status.ERROR,
-                            message = "Failed to load $link"
-                        )
-                    )
-                    continue
-                } else {
-                    if (threadItem.error.isNotBlank()) {
-                        dataTransaction.saveLog(
-                            LogEntryEntity(
-                                type = LogEntryEntity.Type.POST,
-                                status = LogEntryEntity.Status.ERROR,
-                                message = "Error loading $link: ${threadItem.error}"
-                            )
-                        )
-                        continue
-                    } else if (threadItem.postItemList.isEmpty()) {
-                        dataTransaction.saveLog(
-                            LogEntryEntity(
-                                type = LogEntryEntity.Type.POST,
-                                status = LogEntryEntity.Status.ERROR,
-                                message = "Nothing found for $link"
-                            )
-                        )
-                        continue
-                    }
-                }
-
-                val postItem = threadItem.postItemList.firstOrNull { it.postId == postId }
-                if (postItem == null) {
-                    dataTransaction.saveLog(
-                        LogEntryEntity(
-                            type = LogEntryEntity.Type.POST,
-                            status = LogEntryEntity.Status.ERROR,
-                            message = "Unable to load $link"
-                        )
-                    )
-                    continue
-                }
-                toProcess.add(postItem)
-            }
-
-            val posts = try {
-                dataTransaction.newPosts(toProcess.toList())
-            } catch (e: Exception) {
-                e.printStackTrace()
-                emptyList()
-            }
-            posts.forEach {
-                metadataService.fetchMetadata(it.postId)
-            }
-            if (settingsService.settings.downloadSettings.autoStart) {
-                downloadService.restartAll(posts)
-            }
-        } catch (e: Exception) {
-            val error = String.format("Error when adding galleries")
-            log.error(error, e)
-        } finally {
-            Tasks.decrement()
-        }
-    }
-}

vripper-core/src/main/kotlin/me/vripper/tasks/AddPostTask.kt

@@ -0,0 +1,83 @@
+package me.vripper.tasks
+
+import me.vripper.model.ThreadPostId
+import me.vripper.services.*
+import me.vripper.utilities.LoggerDelegate
+import me.vripper.vgapi.PostItem
+import me.vripper.vgapi.PostLookupAPIParser
+import org.koin.core.component.KoinComponent
+import org.koin.core.component.inject
+
+internal class AddPostTask(private val items: List<ThreadPostId>) : KoinComponent, Runnable {
+    private val log by LoggerDelegate()
+    private val dataTransaction: DataTransaction by inject()
+    private val settingsService: SettingsService by inject()
+    private val downloadService: DownloadService by inject()
+    private val threadCacheService: ThreadCacheService by inject()
+    private val metadataService: MetadataService by inject()
+
+    override fun run() {
+        try {
+            Tasks.increment()
+            val toProcess = mutableListOf<PostItem>()
+            for ((threadId, postId) in items) {
+                if (dataTransaction.exists(postId)) {
+                    log.info("Post $postId already loaded")
+                    continue
+                }
+
+                val link =
+                    "${settingsService.settings.viperSettings.host}/threads/$threadId?p=$postId&viewfull=1#post$postId"
+
+                val cachedThread = threadCacheService.getIfPresent(threadId)
+                val threadItem = cachedThread ?:
+                    PostLookupAPIParser(
+                        threadId, postId
+                    ).parse()
+
+
+                if (threadItem == null) {
+                    log.error("Failed to load $link")
+                    continue
+                } else {
+                    if (threadItem.error.isNotBlank()) {
+                        log.error("Error loading $link: ${threadItem.error}")
+                        continue
+                    } else if (threadItem.postItemList.isEmpty()) {
+                        log.error("Nothing found for $link")
+                        continue
+                    }
+                }
+
+                val postItem = threadItem.postItemList.firstOrNull { it.postId == postId }
+                if (postItem == null) {
+                    log.error("Unable to load $link")
+                    continue
+                }
+                toProcess.add(postItem)
+            }
+
+            if (toProcess.isEmpty()) {
+                return
+            }
+
+            val posts = try {
+                dataTransaction.newPosts(toProcess.toList())
+            } catch (e: Exception) {
+                e.printStackTrace()
+                emptyList()
+            }
+            posts.forEach {
+                metadataService.fetchMetadata(it.postId)
+            }
+            if (settingsService.settings.downloadSettings.autoStart) {
+                downloadService.restartAll(posts)
+            }
+        } catch (e: Exception) {
+            val error = String.format("Error when adding galleries")
+            log.error(error, e)
+        } finally {
+            Tasks.decrement()
+        }
+    }
+}

vripper-core/src/main/kotlin/me/vripper/tasks/FetchMetadataTask.kt

@@ -0,0 +1,110 @@
+package me.vripper.tasks
+
+import me.vripper.entities.MetadataEntity
+import me.vripper.exception.DownloadException
+import me.vripper.exception.VripperException
+import me.vripper.services.DataTransaction
+import me.vripper.services.HTTPService
+import me.vripper.services.SettingsService
+import me.vripper.services.VGAuthService
+import me.vripper.utilities.HtmlUtils
+import me.vripper.utilities.LoggerDelegate
+import me.vripper.utilities.RequestLimit
+import me.vripper.utilities.XpathUtils
+import org.apache.hc.client5.http.classic.methods.HttpGet
+import org.apache.hc.core5.http.io.entity.EntityUtils
+import org.apache.hc.core5.net.URIBuilder
+import org.koin.core.component.KoinComponent
+import org.koin.core.component.inject
+import org.w3c.dom.Node
+import java.util.concurrent.atomic.AtomicBoolean
+import java.util.stream.Collectors
+
+internal class FetchMetadataTask(
+    private val postId: Long,
+) : KoinComponent, Runnable {
+    private val dictionary: List<String> = mutableListOf("download", "link", "rapidgator", "filefactory", "filefox")
+    private val log by LoggerDelegate()
+    private val settingsService: SettingsService by inject()
+    private val vgAuthService: VGAuthService by inject()
+    private val httpService: HTTPService by inject()
+    private val dataTransaction: DataTransaction by inject()
+
+    override fun run() {
+        try {
+            if (!settingsService.settings.viperSettings.fetchMetadata) {
+                log.debug("Fetching metadata is disabled")
+                return
+            }
+            Tasks.increment()
+            val httpGet = HttpGet(URIBuilder(settingsService.settings.viperSettings.host + "/threads/").also {
+                it.setParameter(
+                    "p", postId.toString()
+                )
+            }.build())
+
+            RequestLimit.getPermit(1)
+            log.debug("Requesting {}", httpGet.uri)
+            val response = httpService.client.execute(httpGet, vgAuthService.createVgContext()) {
+                if (it.code / 100 != 2) {
+                    throw DownloadException("Unexpected response code '${it.code}' for $httpGet")
+                }
+                EntityUtils.toString(it.entity)
+            }
+
+            val document = HtmlUtils.clean(response)
+            val postNode: Node = XpathUtils.getAsNode(
+                document,
+                "//li[@id='post_$postId']/div[contains(@class, 'postdetails')]",
+
+                ) ?: throw VripperException("Unable to find post #'$postId'")
+
+            val postedBy: String = XpathUtils.getAsNode(
+                postNode, "./div[contains(@class, 'userinfo')]//a[contains(@class, 'username')]//font"
+            )?.textContent?.trim()
+                ?: throw VripperException("Unable to find the poster for post #'$postId'")
+
+
+            val node: Node = XpathUtils.getAsNode(
+                document, java.lang.String.format("//div[@id='post_message_%s']", postId)
+            ) ?: throw VripperException("Unable to locate post content")
+            val titles = findTitleInContent(node)
+            val metadataEntity = MetadataEntity(postId, MetadataEntity.Data(postedBy, titles))
+            dataTransaction.saveMetadata(metadataEntity)
+        } finally {
+            Tasks.decrement()
+        }
+    }
+
+    private fun findTitleInContent(node: Node): List<String> {
+        val altTitle: MutableList<String> = mutableListOf()
+        findTitle(node, altTitle, AtomicBoolean(true))
+        return altTitle.stream().distinct().collect(Collectors.toList())
+    }
+
+    private fun findTitle(node: Node, altTitle: MutableList<String>, keepGoing: AtomicBoolean) {
+        if (!keepGoing.get()) {
+            return
+        }
+        if (node.nodeName == "a" || node.nodeName == "img") {
+            keepGoing.set(false)
+            return
+        }
+        if (node.nodeType == Node.ELEMENT_NODE) {
+            for (i in 0 until node.childNodes.length) {
+                val item = node.childNodes.item(i)
+                findTitle(item, altTitle, keepGoing)
+                if (!keepGoing.get()) {
+                    return
+                }
+            }
+        } else if (node.nodeType == Node.TEXT_NODE) {
+            val text = node.textContent.trim()
+            if (text.isNotBlank() && dictionary.stream().noneMatch { e ->
+                    text.lowercase().contains(e.lowercase())
+                }) {
+                altTitle.add(text)
+            }
+        }
+    }
+}

vripper-core/src/main/kotlin/me/vripper/tasks/LeaveThanksRunnable.kt

@@ -1,71 +0,0 @@
-package me.vripper.tasks
-
-import me.vripper.entities.LogEntryEntity
-import me.vripper.entities.LogEntryEntity.Status.ERROR
-import me.vripper.entities.PostEntity
-import me.vripper.services.DataTransaction
-import me.vripper.services.HTTPService
-import me.vripper.services.SettingsService
-import me.vripper.utilities.Tasks
-import me.vripper.utilities.formatToString
-import org.apache.hc.client5.http.classic.methods.HttpPost
-import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
-import org.apache.hc.client5.http.protocol.HttpClientContext
-import org.apache.hc.core5.http.message.BasicNameValuePair
-import org.koin.core.component.KoinComponent
-import org.koin.core.component.inject
-
-class LeaveThanksRunnable(
-    private val postEntity: PostEntity,
-    private val authenticated: Boolean,
-    private val context: HttpClientContext
-) : KoinComponent, Runnable {
-    private val log by me.vripper.delegate.LoggerDelegate()
-    private val cm: HTTPService by inject()
-    private val settingsService: SettingsService by inject()
-    private val dataTransaction: DataTransaction by inject()
-
-    override fun run() {
-        try {
-            Tasks.increment()
-            if (!settingsService.settings.viperSettings.login || !authenticated) {
-                return
-            }
-            if (!settingsService.settings.viperSettings.thanks) {
-                return
-            }
-
-            val postThanks =
-                HttpPost("${settingsService.settings.viperSettings.host}/post_thanks.php").also {
-                    it.entity = UrlEncodedFormEntity(
-                        listOf(
-                            BasicNameValuePair("do", "post_thanks_add"),
-                            BasicNameValuePair("using_ajax", "1"),
-                            BasicNameValuePair("p", postEntity.postId.toString()),
-                            BasicNameValuePair("securitytoken", postEntity.token)
-                        )
-                    )
-                    it.addHeader("Referer", settingsService.settings.viperSettings.host)
-                    it.addHeader(
-                        "Host",
-                        settingsService.settings.viperSettings.host.replace("https://", "")
-                            .replace("http://", "")
-                    )
-                }
-            cm.client.execute(postThanks, context) { }
-        } catch (e: Exception) {
-            val error = "Failed to leave a thanks for $postEntity"
-            log.error(error, e)
-            dataTransaction.saveLog(
-                LogEntryEntity(
-                    type = LogEntryEntity.Type.THANKS, status = ERROR, message = """
-                $error
-                ${e.formatToString()}
-                """.trimIndent()
-                )
-            )
-        } finally {
-            Tasks.decrement()
-        }
-    }
-}

vripper-core/src/main/kotlin/me/vripper/tasks/LeaveThanksTask.kt

@@ -0,0 +1,57 @@
+package me.vripper.tasks
+
+import me.vripper.entities.PostEntity
+import me.vripper.exception.VripperException
+import me.vripper.services.HTTPService
+import me.vripper.services.SettingsService
+import me.vripper.utilities.LoggerDelegate
+import me.vripper.utilities.RequestLimit
+import org.apache.hc.client5.http.classic.methods.HttpPost
+import org.apache.hc.client5.http.entity.UrlEncodedFormEntity
+import org.apache.hc.client5.http.protocol.HttpClientContext
+import org.apache.hc.core5.http.message.BasicNameValuePair
+import org.koin.core.component.KoinComponent
+import org.koin.core.component.inject
+
+internal class LeaveThanksTask(
+    private val postEntity: PostEntity,
+    private val authenticated: Boolean,
+    private val context: HttpClientContext
+) : KoinComponent, Runnable {
+    private val log by LoggerDelegate()
+    private val cm: HTTPService by inject()
+    private val settingsService: SettingsService by inject()
+
+    override fun run() {
+        try {
+            Tasks.increment()
+            if (!settingsService.settings.viperSettings.login || !authenticated) {
+                return
+            }
+            if (!settingsService.settings.viperSettings.thanks) {
+                return
+            }
+            val postThanks = HttpPost("${settingsService.settings.viperSettings.host}/post_thanks.php").also {
+                it.entity = UrlEncodedFormEntity(
+                    listOf(
+                        BasicNameValuePair("do", "post_thanks_add"),
+                        BasicNameValuePair("using_ajax", "1"),
+                        BasicNameValuePair("p", postEntity.postId.toString()),
+                        BasicNameValuePair("securitytoken", postEntity.token)
+                    )
+                )
+            }
+            RequestLimit.getPermit(1)
+            log.info("Posting {}", postThanks.uri)
+            cm.client.execute(postThanks, context) { response ->
+                if (response.code / 100 != 2) {
+                    throw VripperException("Unexpected response code '${response.code}' for $postThanks")
+                }
+            }
+        } catch (e: Exception) {
+            log.error("Failed to leave a thanks for $postEntity", e)
+        } finally {
+            Tasks.decrement()
+        }
+    }
+}

vripper-core/src/main/kotlin/me/vripper/tasks/Tasks.kt

@@ -0,0 +1,28 @@
+package me.vripper.tasks
+
+import me.vripper.event.EventBus
+import me.vripper.event.LoadingTasks
+import org.koin.core.component.KoinComponent
+import org.koin.core.component.inject
+
+internal object Tasks : KoinComponent {
+
+    private val eventBus: EventBus by inject()
+    private var current = 0
+
+    @Synchronized
+    fun increment() {
+        if (current == 0) {
+            eventBus.publishEvent(LoadingTasks(true))
+        }
+        current += 1
+    }
+
+    @Synchronized
+    fun decrement() {
+        current -= 1
+        if (current == 0) {
+            eventBus.publishEvent(LoadingTasks(false))
+        }
+    }
+}

vripper-core/src/main/kotlin/me/vripper/tasks/ThreadLookupRunnable.kt

@@ -1,87 +0,0 @@
-package me.vripper.tasks
-
-import me.vripper.entities.LogEntryEntity
-import me.vripper.entities.LogEntryEntity.Status.ERROR
-import me.vripper.entities.ThreadEntity
-import me.vripper.model.Settings
-import me.vripper.model.ThreadPostId
-import me.vripper.services.DataTransaction
-import me.vripper.services.SettingsService
-import me.vripper.services.ThreadCacheService
-import me.vripper.utilities.GLOBAL_EXECUTOR
-import me.vripper.utilities.Tasks
-import me.vripper.utilities.formatToString
-import org.koin.core.component.KoinComponent
-import org.koin.core.component.inject
-import java.util.concurrent.CompletableFuture
-
-class ThreadLookupRunnable(private val threadId: Long, private val settings: Settings) : KoinComponent, Runnable {
-    private val log by me.vripper.delegate.LoggerDelegate()
-    private val dataTransaction by inject<DataTransaction>()
-    private val settingsService by inject<SettingsService>()
-    private val threadCacheService by inject<ThreadCacheService>()
-    private val link: String = "${settingsService.settings.viperSettings.host}/threads/$threadId"
-
-    override fun run() {
-        try {
-            Tasks.increment()
-            if (dataTransaction.findThreadByThreadId(threadId).isEmpty) {
-                val threadLookupResult = threadCacheService[threadId]
-                if (threadLookupResult.error.isNotBlank()) {
-                    dataTransaction.saveLog(
-                        LogEntryEntity(
-                            type = LogEntryEntity.Type.THREAD,
-                            status = LogEntryEntity.Status.ERROR,
-                            message = "Error loading $link: ${threadLookupResult.error}"
-                        )
-                    )
-                    return
-                }
-                if (threadLookupResult.postItemList.isEmpty()) {
-                    dataTransaction.saveLog(
-                        LogEntryEntity(
-                            type = LogEntryEntity.Type.THREAD,
-                            status = LogEntryEntity.Status.ERROR,
-                            message = "Nothing found for $link"
-                        )
-                    )
-                    return
-                }
-
-                if (threadLookupResult.postItemList.size <= settings.downloadSettings.autoQueueThreshold) {
-                    CompletableFuture.runAsync(AddPostRunnable(threadLookupResult.postItemList.map {
-                        ThreadPostId(
-                            it.threadId, it.postId
-                        )
-                    }), GLOBAL_EXECUTOR)
-                } else {
-                    try {
-                        dataTransaction.save(
-                            ThreadEntity(
-                                title = threadLookupResult.title,
-                                link = link,
-                                threadId = threadId,
-                                total = threadLookupResult.postItemList.size
-                            )
-                        )
-                    } catch (e: Exception) {
-                        e.printStackTrace()
-                    }
-                }
-            }
-        } catch (e: Exception) {
-            val error = "Error when processing $link"
-            log.error(error, e)
-            dataTransaction.saveLog(
-                LogEntryEntity(
-                    type = LogEntryEntity.Type.THREAD, status = ERROR, message = """
-                $error
-                ${e.formatToString()}
-                """.trimIndent()
-                )
-            )
-        } finally {
-            Tasks.decrement()
-        }
-    }
-}

vripper-core/src/main/kotlin/me/vripper/tasks/ThreadLookupTask.kt

@@ -0,0 +1,60 @@
+package me.vripper.tasks
+
+import me.vripper.entities.ThreadEntity
+import me.vripper.model.Settings
+import me.vripper.model.ThreadPostId
+import me.vripper.services.DataTransaction
+import me.vripper.services.SettingsService
+import me.vripper.services.ThreadCacheService
+import me.vripper.utilities.LoggerDelegate
+import me.vripper.utilities.taskRunner
+import org.koin.core.component.KoinComponent
+import org.koin.core.component.inject
+
+internal class ThreadLookupTask(private val threadId: Long, private val settings: Settings) : KoinComponent, Runnable {
+    private val log by LoggerDelegate()
+    private val dataTransaction by inject<DataTransaction>()
+    private val settingsService by inject<SettingsService>()
+    private val threadCacheService by inject<ThreadCacheService>()
+    private val link: String = "${settingsService.settings.viperSettings.host}/threads/$threadId"
+
+    override fun run() {
+        try {
+            Tasks.increment()
+            if (dataTransaction.findThreadByThreadId(threadId).isEmpty) {
+                val threadLookupResult = threadCacheService[threadId]
+                if (threadLookupResult.error.isNotBlank()) {
+                    log.error("Error loading $link: ${threadLookupResult.error}")
+                    return
+                }
+                if (threadLookupResult.postItemList.isEmpty()) {
+                    log.error("Nothing found for $link")
+                    return
+                }
+
+                if (threadLookupResult.postItemList.size <= settings.downloadSettings.autoQueueThreshold) {
+                    taskRunner.submit(
+                        AddPostTask(threadLookupResult.postItemList.map {
+                            ThreadPostId(
+                                it.threadId, it.postId
+                            )
+                        })
+                    )
+                } else {
+                        dataTransaction.save(
+                            ThreadEntity(
+                                title = threadLookupResult.title,
+                                link = link,
+                                threadId = threadId,
+                                total = threadLookupResult.postItemList.size
+                            )
+                        )
+                }
+            }
+        } catch (e: Exception) {
+            log.error("Error when processing $link", e)
+        } finally {
+            Tasks.decrement()
+        }
+    }
+}

vripper-core/src/main/kotlin/me/vripper/utilities/AesUtils.kt

@@ -0,0 +1,72 @@
+package me.vripper.utilities
+
+import java.nio.charset.StandardCharsets
+import java.security.MessageDigest
+import java.security.SecureRandom
+import java.util.concurrent.locks.ReentrantLock
+import javax.crypto.Cipher
+import javax.crypto.spec.GCMParameterSpec
+import javax.crypto.spec.SecretKeySpec
+import kotlin.concurrent.withLock
+import kotlin.io.encoding.ExperimentalEncodingApi
+
+
+@OptIn(ExperimentalEncodingApi::class)
+object AesUtils {
+
+    val mutex = ReentrantLock()
+
+    val secureRandom: SecureRandom = SecureRandom()
+    val cipher: Cipher = Cipher.getInstance(AES_ALGORITHM_GCM)
+
+    const val SHA_CRYPT: String = "SHA-256"
+    const val AES_ALGORITHM: String = "AES"
+    const val AES_ALGORITHM_GCM: String = "AES/GCM/NoPadding"
+    const val IV_LENGTH_ENCRYPT: Int = 12
+    const val TAG_LENGTH_ENCRYPT: Int = 16
+
+    fun aesDecrypt(data: ByteArray, passPhrase: String): ByteArray {
+
+        val key = generateAesKeyFromPassphrase(passPhrase)
+
+        val iv = ByteArray(IV_LENGTH_ENCRYPT)
+        System.arraycopy(data, 0, iv, 0, iv.size)
+        val encryptedText = ByteArray(data.size - IV_LENGTH_ENCRYPT)
+        System.arraycopy(data, IV_LENGTH_ENCRYPT, encryptedText, 0, encryptedText.size)
+
+        val gcmSpec = GCMParameterSpec(TAG_LENGTH_ENCRYPT * 8, iv)
+        val decryptedBytes = mutex.withLock {
+            cipher.init(Cipher.DECRYPT_MODE, key, gcmSpec)
+            cipher.doFinal(encryptedText)
+        }
+
+        return decryptedBytes
+    }
+
+    fun aesEncrypt(data: ByteArray, passPhrase: String): ByteArray {
+
+        val iv = ByteArray(IV_LENGTH_ENCRYPT)
+        secureRandom.nextBytes(iv)
+
+        val key = generateAesKeyFromPassphrase(passPhrase)
+
+        val gcmSpec = GCMParameterSpec(TAG_LENGTH_ENCRYPT * 8, iv)
+        val encryptedBytes = mutex.withLock {
+            cipher.init(Cipher.ENCRYPT_MODE, key, gcmSpec)
+            cipher.doFinal(data)
+        }
+
+        // Combine IV and encrypted text and encode them as Base64
+        val combinedIvAndCipherText = ByteArray(iv.size + encryptedBytes.size)
+        System.arraycopy(iv, 0, combinedIvAndCipherText, 0, iv.size)
+        System.arraycopy(encryptedBytes, 0, combinedIvAndCipherText, iv.size, encryptedBytes.size)
+
+        return combinedIvAndCipherText
+    }
+
+    fun generateAesKeyFromPassphrase(passPhrase: String): SecretKeySpec {
+        val sha256 = MessageDigest.getInstance(SHA_CRYPT)
+        val keyBytes = sha256.digest(passPhrase.toByteArray(StandardCharsets.UTF_8))
+        return SecretKeySpec(keyBytes, AES_ALGORITHM)
+    }
+}

vripper-core/src/main/kotlin/me/vripper/utilities/AppLock.kt

@@ -1,4 +1,4 @@
-package me.vripper.listeners
+package me.vripper.utilities
 
 import me.vripper.utilities.ApplicationProperties.VRIPPER_DIR
 import java.io.RandomAccessFile
@@ -7,24 +7,25 @@ import kotlin.io.path.pathString
 import kotlin.system.exitProcess
 
 object AppLock {
-    fun exclusiveLock() {
+    fun exclusiveLock(): Boolean {
         val lock = VRIPPER_DIR.resolve("lock")
         try {
             val randomFile = RandomAccessFile(lock.pathString, "rw")
             val channel = randomFile.channel
             val fileLock = channel.tryLock()
             if (fileLock == null) {
-                System.err.println("Another instance is already running in ${lock.parent.pathString}")
-                exitProcess(-1)
+                return false
             } else {
                 Runtime.getRuntime().addShutdownHook(Thread {
                     fileLock.release()
                     channel.close()
                     Files.deleteIfExists(lock)
                 })
+                return true
             }
         } catch (e: Exception) {
-            println(e.toString())
+            e.printStackTrace()
+            exitProcess(-1)
         }
     }
 }

vripper-core/src/main/kotlin/me/vripper/utilities/ApplicationProperties.kt

@@ -11,7 +11,9 @@ import java.net.http.HttpResponse
 import java.nio.file.Files
 import java.nio.file.Path
 import kotlin.io.path.Path
+import kotlin.io.path.exists
 import kotlin.io.path.pathString
+import kotlin.io.path.readLines
 
 object ApplicationProperties {
     val VERSION: String =
@@ -24,6 +26,8 @@ object ApplicationProperties {
     private val json = Json {
         ignoreUnknownKeys = true
     }
+    const val DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:136.0) Gecko/20100101 Firefox/136.0"
+    val USER_AGENT: String;
 
     @Serializable
     internal data class ReleaseResponse(@SerialName("tag_name") val tagName: String)
@@ -31,11 +35,17 @@ object ApplicationProperties {
     init {
         Files.createDirectories(VRIPPER_DIR)
         System.setProperty("VRIPPER_DIR", VRIPPER_DIR.toRealPath().pathString)
+        val userAgentOverride = VRIPPER_DIR.resolve("user-agent")
+        USER_AGENT = if (userAgentOverride.exists()) {
+            userAgentOverride.readLines().firstOrNull() ?: DEFAULT_USER_AGENT
+        } else {
+            DEFAULT_USER_AGENT
+        }
     }
 
     fun latestVersion(): String {
         val request = HttpRequest.newBuilder()
-            .uri(URI.create("https://api.github.com/repos/death-claw/vripper-project/releases/latest")).build()
+            .uri(URI.create("https://api.github.com/repos/dev-claw/vripper-project/releases/latest")).build()
         return HttpClient.newHttpClient().use {
             val response = it.send(request, HttpResponse.BodyHandlers.ofString())
             if (response.statusCode() / 100 != 2) {

vripper-core/src/main/kotlin/me/vripper/utilities/DatabaseManager.kt

@@ -3,21 +3,24 @@ package me.vripper.utilities
 import liquibase.Contexts
 import liquibase.LabelExpression
 import liquibase.Liquibase
-import liquibase.database.Database
 import liquibase.database.DatabaseFactory
 import liquibase.database.jvm.JdbcConnection
 import liquibase.resource.ClassLoaderResourceAccessor
-import me.vripper.utilities.ApplicationProperties.VRIPPER_DIR
-import java.sql.DriverManager
+import org.jetbrains.exposed.sql.Database
+import java.sql.Connection
 
-object DbUtils {
+object DatabaseManager {
+    private var database: Database? = null
 
-    fun update() {
-        val connection =
-            DriverManager.getConnection("jdbc:h2:file:$VRIPPER_DIR/vripper;DB_CLOSE_DELAY=-1;")
+    fun connect() {
+        database =
+            Database.connect("jdbc:sqlite:${ApplicationProperties.VRIPPER_DIR}/vripper.db")
+                .also { update(it.connector.invoke().connection as Connection) }
+    }
 
+    private fun update(connection: Connection) {
         connection.use { cn ->
-            val database: Database = DatabaseFactory.getInstance()
+            val database: liquibase.database.Database = DatabaseFactory.getInstance()
                 .findCorrectDatabaseImplementation(JdbcConnection(cn))
             Liquibase(
                 "db.changelog-master.xml", ClassLoaderResourceAccessor(), database

vripper-core/src/main/kotlin/me/vripper/utilities/Executors.kt

@@ -0,0 +1,7 @@
+package me.vripper.utilities
+
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+
+val taskRunner: ExecutorService = Executors.newFixedThreadPool(6)
+val downloadRunner: ExecutorService = Executors.newVirtualThreadPerTaskExecutor()

vripper-core/src/main/kotlin/me/vripper/utilities/HtmlUtils.kt

@@ -7,11 +7,12 @@ import org.htmlcleaner.HtmlCleaner
 import org.w3c.dom.Document
 import java.io.InputStream
 
-object HtmlUtils {
+internal object HtmlUtils {
+
     @Throws(HtmlProcessorException::class)
     fun clean(htmlContent: InputStream): Document {
         return try {
-            htmlContent.use {
+            htmlContent.use { stream ->
                 val clean = HtmlCleaner().clean(htmlContent)
                 DomSerializer(CleanerProperties()).createDOM(clean)
             }
@@ -19,4 +20,15 @@ object HtmlUtils {
             throw HtmlProcessorException(e)
         }
     }
+
+    @Throws(HtmlProcessorException::class)
+    fun clean(htmlContent: String): Document {
+        return try {
+            val clean = HtmlCleaner().clean(htmlContent)
+            DomSerializer(CleanerProperties()).createDOM(clean)
+
+        } catch (e: Exception) {
+            throw HtmlProcessorException(e)
+        }
+    }
 }

vripper-core/src/main/kotlin/me/vripper/utilities/LoggerDelegate.kt

@@ -1,4 +1,4 @@
-package me.vripper.delegate
+package me.vripper.utilities
 
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory

vripper-core/src/main/kotlin/me/vripper/utilities/MD5.kt

@@ -0,0 +1,10 @@
+package me.vripper.utilities
+
+import java.security.MessageDigest
+import javax.xml.bind.DatatypeConverter
+
+fun md5Hex(data: String): String {
+    val md: MessageDigest = MessageDigest.getInstance("MD5")
+    md.update(data.toByteArray())
+    return DatatypeConverter.printHexBinary(md.digest()).lowercase()
+}

vripper-core/src/main/kotlin/me/vripper/utilities/PathUtils.kt

@@ -11,8 +11,8 @@ import kotlin.io.path.Path
 import kotlin.io.path.listDirectoryEntries
 
 
-object PathUtils {
-    private val log by me.vripper.delegate.LoggerDelegate()
+internal object PathUtils {
+    private val log by LoggerDelegate()
     private val imageExtensions = listOf("bmp", "gif", "jpg", "jpeg", "png", "webp")
 
 
@@ -54,7 +54,7 @@ object PathUtils {
                 try {
                     Files.delete(currentDownloadDirectory)
                 } catch (e: Exception) {
-                    e.printStackTrace()
+                    log.error("Failed to delete directory $currentDownloadDirectory", e)
                 }
             }
         } catch (e: IOException) {

vripper-core/src/main/kotlin/me/vripper/utilities/RequestLimit.kt

@@ -1,7 +1,42 @@
 package me.vripper.utilities
 
-import kotlinx.coroutines.sync.Semaphore
+import io.github.bucket4j.Bucket
+import io.github.bucket4j.BucketConfiguration
+import io.github.bucket4j.TokensInheritanceStrategy
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.SupervisorJob
+import kotlinx.coroutines.flow.collectLatest
+import kotlinx.coroutines.flow.filterIsInstance
+import kotlinx.coroutines.launch
+import me.vripper.event.EventBus
+import me.vripper.event.SettingsUpdateEvent
+import me.vripper.services.SettingsService
+import org.koin.core.component.KoinComponent
+import org.koin.core.component.inject
+import java.time.Duration
 
-object RequestLimit {
-    val semaphore: Semaphore = Semaphore(6)
+internal object RequestLimit : KoinComponent {
+    private val coroutineScope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
+    private val settingsService: SettingsService by inject()
+    private val eventBus: EventBus by inject()
+    private val bucket: Bucket = Bucket.builder().addLimit {
+        it.capacity(settingsService.settings.viperSettings.requestLimit)
+            .refillGreedy(settingsService.settings.viperSettings.requestLimit, Duration.ofSeconds(1))
+    }.build()
+
+    init {
+        coroutineScope.launch {
+            eventBus.events.filterIsInstance(SettingsUpdateEvent::class).collectLatest { event ->
+                bucket.replaceConfiguration(BucketConfiguration.builder().addLimit {
+                    it.capacity(event.settings.viperSettings.requestLimit)
+                        .refillGreedy(event.settings.viperSettings.requestLimit, Duration.ofSeconds(1))
+                }.build(), TokensInheritanceStrategy.RESET)
+            }
+        }
+    }
+
+    fun getPermit(tokens: Long) {
+        bucket.asBlocking().consume(tokens)
+    }
 }

vripper-core/src/main/kotlin/me/vripper/utilities/StackTraceElementUtils.kt

@@ -0,0 +1,24 @@
+package me.vripper.utilities
+
+import ch.qos.logback.classic.spi.IThrowableProxy
+
+object StackTraceElementUtils {
+    fun format(throwable: IThrowableProxy): String {
+        val sb = StringBuilder()
+        sb.appendLine(throwable.className + ": " + throwable.message)
+        stackTraceElementFormat(throwable, sb)
+        return sb.toString()
+
+    }
+
+    private fun stackTraceElementFormat(throwable: IThrowableProxy, sb: StringBuilder) {
+        throwable.stackTraceElementProxyArray.forEach {
+            sb.appendLine("\t" + it.steAsString)
+        }
+
+        if (throwable.cause != null) {
+            sb.appendLine("Caused by: " + throwable.cause.className + ": " + throwable.cause.message)
+            stackTraceElementFormat(throwable.cause, sb)
+        }
+    }
+}

vripper-core/src/main/kotlin/me/vripper/utilities/Tasks.kt

@@ -1,37 +0,0 @@
-package me.vripper.utilities
-
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.SupervisorJob
-import kotlinx.coroutines.launch
-import me.vripper.event.EventBus
-import me.vripper.event.LoadingTasks
-import org.koin.core.component.KoinComponent
-import org.koin.core.component.inject
-
-object Tasks : KoinComponent {
-
-    private val eventBus: EventBus by inject()
-    private var current = 0
-    private val coroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
-
-    @Synchronized
-    fun increment() {
-        if (current == 0) {
-            coroutineScope.launch {
-                eventBus.publishEvent(LoadingTasks(true))
-            }
-        }
-        current += 1
-    }
-
-    @Synchronized
-    fun decrement() {
-        current -= 1
-        if (current == 0) {
-            coroutineScope.launch {
-                eventBus.publishEvent(LoadingTasks(false))
-            }
-        }
-    }
-}

vripper-core/src/main/kotlin/me/vripper/utilities/ThreadPool.kt

@@ -1,6 +0,0 @@
-package me.vripper.utilities
-
-import java.util.concurrent.ExecutorService
-import java.util.concurrent.Executors
-
-val GLOBAL_EXECUTOR: ExecutorService = Executors.newCachedThreadPool()

vripper-core/src/main/kotlin/me/vripper/utilities/XpathUtils.kt

@@ -6,7 +6,7 @@ import org.w3c.dom.NodeList
 import javax.xml.xpath.XPathConstants
 import javax.xml.xpath.XPathFactory
 
-object XpathUtils {
+internal object XpathUtils {
     private val xpath = XPathFactory.newInstance().newXPath()
 
     @Throws(XpathException::class)

vripper-core/src/main/kotlin/me/vripper/vgapi/ImageItem.kt

@@ -0,0 +1,5 @@
+package me.vripper.vgapi
+
+import me.vripper.host.Host
+
+internal data class ImageItem(val mainLink: String, val thumbLink: String, val host: Host)

vripper-core/src/main/kotlin/me/vripper/vgapi/PostItem.kt

@@ -1,6 +1,6 @@
-package me.vripper.parser
+package me.vripper.vgapi
 
-data class PostItem(
+internal data class PostItem(
     val threadId: Long,
     val threadTitle: String,
     val postId: Long,

vripper-core/src/main/kotlin/me/vripper/vgapi/PostLookupAPIParser.kt

@@ -0,0 +1,71 @@
+package me.vripper.vgapi
+
+import dev.failsafe.Failsafe
+import dev.failsafe.function.CheckedSupplier
+import me.vripper.exception.DownloadException
+import me.vripper.exception.PostParseException
+import me.vripper.services.HTTPService
+import me.vripper.services.RetryPolicyService
+import me.vripper.services.VGAuthService
+import me.vripper.tasks.Tasks
+import me.vripper.utilities.LoggerDelegate
+import me.vripper.utilities.RequestLimit
+import org.apache.hc.client5.http.classic.methods.HttpGet
+import org.apache.hc.core5.http.io.entity.EntityUtils
+import org.apache.hc.core5.net.URIBuilder
+import org.koin.core.component.KoinComponent
+import org.koin.core.component.inject
+import java.io.ByteArrayInputStream
+import javax.xml.parsers.SAXParserFactory
+
+internal class PostLookupAPIParser(private val threadId: Long, private val postId: Long) : KoinComponent {
+    private val log by LoggerDelegate()
+    private val retryPolicyService: RetryPolicyService by inject()
+    private val httpService: HTTPService by inject()
+    private val vgAuthService: VGAuthService by inject()
+
+    @Throws(PostParseException::class)
+    fun parse(): ThreadItem? {
+        log.debug("Parsing post $postId")
+        val httpGet =
+            HttpGet(URIBuilder("https://viper.click/vr.php").also {
+                it.setParameter(
+                    "p", postId.toString()
+                )
+            }.build())
+        val threadLookupAPIResponseHandler = ThreadLookupAPIResponseHandler()
+        Tasks.increment()
+        return try {
+            Failsafe.with(retryPolicyService.buildRetryPolicy<Any>("Failed to parse $httpGet: ")).onFailure {
+                log.error(
+                    "Failed to process thread $threadId, post $postId", it.exception
+                )
+            }.get(CheckedSupplier {
+                RequestLimit.getPermit(1)
+                log.info("Requesting {}", httpGet.uri)
+                httpService.client.execute(
+                    httpGet,
+                    vgAuthService.createClickContext()
+                ) { response ->
+                    if (response.code / 100 != 2) {
+                        throw DownloadException("Unexpected response code '${response.code}' for $httpGet")
+                    }
+                    ByteArrayInputStream(EntityUtils.toByteArray(response.entity)).use {
+                        factory.newSAXParser()
+                            .parse(it, threadLookupAPIResponseHandler)
+                        threadLookupAPIResponseHandler.result
+                    }
+                }
+
+            })
+        } catch (e: Exception) {
+            throw PostParseException(e)
+        } finally {
+            Tasks.decrement()
+        }
+    }
+
+    companion object {
+        private val factory = SAXParserFactory.newInstance()
+    }
+}

vripper-core/src/main/kotlin/me/vripper/vgapi/ThreadItem.kt

@@ -1,6 +1,6 @@
-package me.vripper.parser
+package me.vripper.vgapi
 
-data class ThreadItem(
+internal data class ThreadItem(
     val threadId: Long,
     val title: String,
     val securityToken: String,