v3.3.0

Add a Event Logs
Fix bugs with the database

CHANGELOG.md

@@ -1,11 +1,23 @@
 # Changelog
 
-## [3.2.3] - 2021-01-13
+## [3.3.0] - 2021-02-16
+
 ### Changed
+
+- Add support for proxies
+- Add a Event Logs
+- Fix bugs with the database
+
+## [3.2.3] - 2021-01-13
+
+### Changed
+
 - Finished posts are not cleared from the UI
 
 ## [3.2.2] - 2021-01-13
+
 ### Changed
+
 - Set total download to max value
 
 ## [3.2.1] - 2021-01-12

pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>3.2.3</version>
+    <version>3.3.0</version>
     <packaging>pom</packaging>
     <parent>
         <groupId>org.springframework.boot</groupId>

vripper-electron/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-electron",
-  "version": "3.2.3",
+  "version": "3.3.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

vripper-electron/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-electron",
-  "version": "3.2.3",
+  "version": "3.3.0",
   "description": "A ripper for vipergirls.to built using web technolgies",
   "main": "main.js",
   "author": "death-claw <53543762+death-claw@users.noreply.github.com>",

vripper-electron/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>tn.mnlr</groupId>
         <artifactId>vripper</artifactId>
-        <version>3.2.3</version>
+        <version>3.3.0</version>
     </parent>
     <artifactId>vripper-electron</artifactId>
     <name>vripper-electron</name>

vripper-server/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>tn.mnlr</groupId>
         <artifactId>vripper</artifactId>
-        <version>3.2.3</version>
+        <version>3.3.0</version>
     </parent>
     <artifactId>vripper-server</artifactId>
     <name>vripper-server</name>

vripper-server/src/main/java/tn/mnlr/vripper/EventListenerBean.java

@@ -27,10 +27,7 @@ public class EventListenerBean {
 
     @EventListener
     public void onApplicationEvent(ContextRefreshedEvent event) {
-        repositorySet.forEach(IRepository::init);
         dataService.setDownloadingToStopped();
         init = true;
     }
-
-
-}
+}

vripper-server/src/main/java/tn/mnlr/vripper/Utils.java

@@ -0,0 +1,17 @@
+package tn.mnlr.vripper;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+
+public class Utils {
+
+    public static String throwableToString(Throwable th) throws IOException {
+        try (StringWriter stringWriter = new StringWriter(); PrintWriter printWriter = new PrintWriter(stringWriter)) {
+            th.printStackTrace(printWriter);
+            printWriter.flush();
+            stringWriter.flush();
+            return stringWriter.toString();
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadJob.java

@@ -45,7 +45,7 @@ public class DownloadJob implements CheckedRunnable {
     private final Image image;
     @Getter
     private final Post post;
-    private boolean stopped = false;
+    private volatile boolean stopped = false;
     @Getter
     private boolean finished = false;
 

vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadRunnable.java

@@ -3,16 +3,22 @@ package tn.mnlr.vripper.download;
 import lombok.extern.slf4j.Slf4j;
 import net.jodah.failsafe.Failsafe;
 import tn.mnlr.vripper.SpringContext;
+import tn.mnlr.vripper.Utils;
+import tn.mnlr.vripper.jpa.domain.Event;
 import tn.mnlr.vripper.jpa.domain.enums.Status;
+import tn.mnlr.vripper.jpa.repositories.IEventRepository;
 import tn.mnlr.vripper.services.ConnectionService;
 import tn.mnlr.vripper.services.DataService;
 
+import java.time.LocalDateTime;
+
 @Slf4j
 public class DownloadRunnable implements Runnable {
 
     private final DownloadService downloadService;
     private final DataService dataService;
     private final ConnectionService connectionService;
+    private final IEventRepository eventRepository;
 
     private final DownloadJob downloadJob;
 
@@ -20,6 +26,8 @@ public class DownloadRunnable implements Runnable {
         downloadService = SpringContext.getBean(DownloadService.class);
         dataService = SpringContext.getBean(DataService.class);
         connectionService = SpringContext.getBean(ConnectionService.class);
+        eventRepository = SpringContext.getBean(IEventRepository.class);
+
         this.downloadJob = downloadJob;
     }
 
@@ -27,6 +35,12 @@ public class DownloadRunnable implements Runnable {
     public void run() {
         Failsafe.with(connectionService.getRetryPolicy())
                 .onFailure(e -> {
+                    try {
+                        Event event = new Event(Event.Type.DOWNLOAD, Event.Status.ERROR, LocalDateTime.now(), String.format("Failed to download %s\n %s", downloadJob.getImage().getUrl(), Utils.throwableToString(e.getFailure())));
+                        eventRepository.save(event);
+                    } catch (Exception exp) {
+                        log.error("Failed to save event", exp);
+                    }
                     log.error(String.format("Failed to download %s after %d tries", downloadJob.getImage().getUrl(), e.getAttemptCount()), e.getFailure());
                     downloadJob.getImage().setStatus(Status.ERROR);
                     dataService.updateImageStatus(downloadJob.getImage().getStatus(), downloadJob.getImage().getId());

vripper-server/src/main/java/tn/mnlr/vripper/event/EventRemoveEvent.java

@@ -0,0 +1,15 @@
+package tn.mnlr.vripper.event;
+
+import lombok.Getter;
+import org.springframework.context.ApplicationEvent;
+
+@Getter
+public class EventRemoveEvent extends ApplicationEvent {
+
+    private final Long id;
+
+    public EventRemoveEvent(Object source, Long id) {
+        super(source);
+        this.id = id;
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/event/EventUpdateEvent.java

@@ -0,0 +1,15 @@
+package tn.mnlr.vripper.event;
+
+import lombok.Getter;
+import org.springframework.context.ApplicationEvent;
+
+@Getter
+public class EventUpdateEvent extends ApplicationEvent {
+
+    private final Long id;
+
+    public EventUpdateEvent(Object source, Long id) {
+        super(source);
+        this.id = id;
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/event/MetadataUpdateEvent.java

@@ -6,12 +6,10 @@ import org.springframework.context.ApplicationEvent;
 @Getter
 public class MetadataUpdateEvent extends ApplicationEvent {
 
-    private final Long id;
     private final Long postIdRef;
 
-    public MetadataUpdateEvent(Object source, Long id, Long postIdRef) {
+    public MetadataUpdateEvent(Object source, Long postIdRef) {
         super(source);
-        this.id = id;
         this.postIdRef = postIdRef;
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/host/ImxHost.java

@@ -68,6 +68,9 @@ public class ImxHost extends Host {
         try {
             log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url));
             contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH);
+            if (contDiv == null) {
+                throw new HostException(CONTINUE_BUTTON_XPATH + " cannot be found");
+            }
             Node node = contDiv.getAttributes().getNamedItem("value");
             if (node != null) {
                 value = node.getTextContent();

vripper-server/src/main/java/tn/mnlr/vripper/jpa/domain/Event.java

@@ -0,0 +1,64 @@
+package tn.mnlr.vripper.jpa.domain;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.ToString;
+
+import java.io.IOException;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+
+@Getter
+@Setter
+@ToString
+@NoArgsConstructor
+public class Event {
+
+    private Long id;
+    private Type type;
+    private Status status;
+    @JsonSerialize(using = DateTimeSerializer.class)
+    private LocalDateTime time;
+    private String message;
+
+    public Event(Type type, Status status, LocalDateTime time, String message) {
+        this.type = type;
+        this.status = status;
+        this.time = time;
+        this.message = message;
+    }
+
+    public enum Type {
+        POST,
+        QUEUED,
+        THANKS,
+        METADATA,
+        SCAN,
+        DOWNLOAD,
+        METADATA_CACHE_MISS,
+        QUEUED_CACHE_MISS
+    }
+
+    public enum Status {
+        PENDING, PROCESSING, DONE, ERROR
+    }
+}
+
+class DateTimeSerializer extends StdSerializer<LocalDateTime> {
+
+    private final static DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss.SSS");
+
+    protected DateTimeSerializer() {
+        super(LocalDateTime.class);
+    }
+
+    @Override
+    public void serialize(LocalDateTime value, JsonGenerator gen, SerializerProvider provider) throws IOException {
+        gen.writeString(value.format(DATE_TIME_FORMATTER));
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/jpa/domain/Metadata.java

@@ -1,6 +1,5 @@
 package tn.mnlr.vripper.jpa.domain;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
@@ -13,12 +12,11 @@ import java.util.List;
 @NoArgsConstructor
 public class Metadata {
 
-    @JsonIgnore
-    private Long id;
+    private Long postIdRef;
 
-    private List<String> resolvedNames = Collections.emptyList();
+    private String PostId;
 
     private String postedBy;
 
-    private Long postIdRef;
+    private List<String> resolvedNames = Collections.emptyList();
 }

vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IEventRepository.java

@@ -0,0 +1,21 @@
+package tn.mnlr.vripper.jpa.repositories;
+
+import tn.mnlr.vripper.jpa.domain.Event;
+
+import java.util.List;
+import java.util.Optional;
+
+public interface IEventRepository extends IRepository {
+
+    Event save(Event event);
+
+    Event update(Event event);
+
+    List<Event> findAll();
+
+    Optional<Event> findById(Long id);
+
+    void delete(Long id);
+
+    void deleteAll();
+}

vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IMetadataRepository.java

@@ -8,8 +8,6 @@ public interface IMetadataRepository extends IRepository {
 
     Metadata save(Metadata metadata);
 
-    Optional<Metadata> findById(Long id);
-
     Optional<Metadata> findByPostId(String postId);
 
     int deleteByPostId(String postId);

vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IQueuedRepository.java

@@ -15,4 +15,6 @@ public interface IQueuedRepository extends IRepository {
     Optional<Queued> findById(Long id);
 
     int deleteByThreadId(String threadId);
+
+    void deleteAll();
 }

vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IRepository.java

@@ -1,6 +1,4 @@
 package tn.mnlr.vripper.jpa.repositories;
 
 public interface IRepository {
-
-    void init();
 }

vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/EventRepository.java

@@ -0,0 +1,117 @@
+package tn.mnlr.vripper.jpa.repositories.impl;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.context.ApplicationEventPublisher;
+import org.springframework.context.ApplicationEventPublisherAware;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.lang.NonNull;
+import org.springframework.stereotype.Service;
+import tn.mnlr.vripper.event.EventRemoveEvent;
+import tn.mnlr.vripper.event.EventUpdateEvent;
+import tn.mnlr.vripper.jpa.domain.Event;
+import tn.mnlr.vripper.jpa.repositories.IEventRepository;
+import tn.mnlr.vripper.services.SettingsService;
+
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+import java.util.Optional;
+
+@Service
+@Slf4j
+public class EventRepository implements IEventRepository, ApplicationEventPublisherAware {
+
+    private final JdbcTemplate jdbcTemplate;
+    private final SettingsService settingsService;
+    private ApplicationEventPublisher applicationEventPublisher;
+
+    public EventRepository(JdbcTemplate jdbcTemplate, SettingsService settingsService) {
+        this.jdbcTemplate = jdbcTemplate;
+        this.settingsService = settingsService;
+    }
+
+    private synchronized Long nextId() {
+        return jdbcTemplate.queryForObject(
+                "CALL NEXT VALUE FOR SEQ_EVENT",
+                Long.class);
+    }
+
+    @Override
+    public synchronized Event save(@NonNull Event event) {
+
+        int maxRecords = settingsService.getSettings().getMaxEventLog() - 1;
+
+        Long count = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM EVENT", Long.class);
+        if (count > maxRecords) {
+            List<Long> idList = jdbcTemplate.queryForList("SELECT ID FROM EVENT ORDER BY TIME ASC LIMIT ?", Long.class, count - maxRecords);
+            idList.forEach(this::delete);
+        }
+
+        long id = nextId();
+        jdbcTemplate.update(
+                "INSERT INTO EVENT (ID, TYPE, STATUS, TIME, MESSAGE) VALUES (?,?,?,?,?)",
+                id,
+                event.getType().name(),
+                event.getStatus().name(),
+                event.getTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
+                event.getMessage()
+        );
+        event.setId(id);
+        applicationEventPublisher.publishEvent(new EventUpdateEvent(EventRepository.class, id));
+        return event;
+    }
+
+    @Override
+    public Event update(@NonNull Event event) {
+        if (event.getId() == null) {
+            log.warn("Cannot update entity with null id");
+            return event;
+        }
+
+        jdbcTemplate.update(
+                "UPDATE EVENT SET TYPE = ?, STATUS = ?, TIME = ?, MESSAGE = ? WHERE ID = ?",
+                event.getType().name(),
+                event.getStatus().name(),
+                event.getTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME),
+                event.getMessage(),
+                event.getId()
+        );
+        applicationEventPublisher.publishEvent(new EventUpdateEvent(EventRepository.class, event.getId()));
+        return event;
+    }
+
+    @Override
+    public Optional<Event> findById(Long id) {
+        List<Event> events = jdbcTemplate.query(
+                "SELECT * FROM EVENT WHERE ID = ?",
+                new EventRowMapper(),
+                id);
+        if (events.isEmpty()) {
+            return Optional.empty();
+        } else {
+            return Optional.of(events.get(0));
+        }
+    }
+
+    @Override
+    public List<Event> findAll() {
+        return jdbcTemplate.query(
+                "SELECT * FROM EVENT",
+                new EventRowMapper());
+    }
+
+    @Override
+    public void delete(Long id) {
+        jdbcTemplate.update("DELETE FROM EVENT WHERE ID = ?", id);
+        applicationEventPublisher.publishEvent(new EventRemoveEvent(EventRepository.class, id));
+    }
+
+    @Override
+    public void deleteAll() {
+        jdbcTemplate.update("DELETE FROM EVENT");
+    }
+
+    @Override
+    public void setApplicationEventPublisher(@NonNull ApplicationEventPublisher applicationEventPublisher) {
+        this.applicationEventPublisher = applicationEventPublisher;
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/EventRowMapper.java

@@ -0,0 +1,23 @@
+package tn.mnlr.vripper.jpa.repositories.impl;
+
+import org.springframework.jdbc.core.RowMapper;
+import tn.mnlr.vripper.jpa.domain.Event;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+
+public class EventRowMapper implements RowMapper<Event> {
+
+    @Override
+    public Event mapRow(ResultSet rs, int rowNum) throws SQLException {
+        Event event = new Event();
+        event.setId(rs.getLong("ID"));
+        event.setType(Event.Type.valueOf(rs.getString("TYPE")));
+        event.setStatus(Event.Status.valueOf(rs.getString("STATUS")));
+        event.setTime(LocalDateTime.parse(rs.getString("TIME"), DateTimeFormatter.ISO_LOCAL_DATE_TIME));
+        event.setMessage(rs.getString("MESSAGE"));
+        return event;
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/ImageRepository.java

@@ -4,6 +4,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationEventPublisher;
 import org.springframework.context.ApplicationEventPublisherAware;
 import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.lang.NonNull;
 import org.springframework.stereotype.Service;
 import tn.mnlr.vripper.event.ImageUpdateEvent;
 import tn.mnlr.vripper.jpa.domain.Image;
@@ -12,13 +13,11 @@ import tn.mnlr.vripper.jpa.repositories.IImageRepository;
 
 import java.util.List;
 import java.util.Optional;
-import java.util.concurrent.atomic.AtomicLong;
 
 @Service
 public class ImageRepository implements IImageRepository, ApplicationEventPublisherAware {
 
     private final JdbcTemplate jdbcTemplate;
-    private final AtomicLong counter = new AtomicLong(0);
     private ApplicationEventPublisher applicationEventPublisher;
 
     @Autowired
@@ -26,21 +25,15 @@ public class ImageRepository implements IImageRepository, ApplicationEventPublis
         this.jdbcTemplate = jdbcTemplate;
     }
 
-    @Override
-    public void init() {
-        Long maxId = jdbcTemplate.queryForObject(
-                "SELECT MAX(ID) FROM IMAGE",
-                Long.class
-        );
-        if (maxId == null) {
-            maxId = 0L;
-        }
-        counter.set(maxId);
+    private synchronized Long nextId() {
+        return jdbcTemplate.queryForObject(
+                "CALL NEXT VALUE FOR SEQ_IMAGE",
+                Long.class);
     }
 
     @Override
     public Image save(Image image) {
-        long id = counter.incrementAndGet();
+        long id = nextId();
         jdbcTemplate.update(
                 "INSERT INTO IMAGE (ID, CURRENT, HOST, INDEX, POST_ID, STATUS, TOTAL, URL, POST_ID_REF) VALUES (?,?,?,?,?,?,?,?,?)",
                 id,
@@ -147,7 +140,7 @@ public class ImageRepository implements IImageRepository, ApplicationEventPublis
     }
 
     @Override
-    public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
+    public void setApplicationEventPublisher(@NonNull ApplicationEventPublisher applicationEventPublisher) {
         this.applicationEventPublisher = applicationEventPublisher;
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/MetadataRepository.java

@@ -11,13 +11,11 @@ import tn.mnlr.vripper.jpa.repositories.IMetadataRepository;
 
 import java.util.List;
 import java.util.Optional;
-import java.util.concurrent.atomic.AtomicLong;
 
 @Service
 public class MetadataRepository implements IMetadataRepository, ApplicationEventPublisherAware {
 
     private final JdbcTemplate jdbcTemplate;
-    private final AtomicLong counter = new AtomicLong(0);
     private ApplicationEventPublisher applicationEventPublisher;
 
     @Autowired
@@ -25,53 +23,25 @@ public class MetadataRepository implements IMetadataRepository, ApplicationEvent
         this.jdbcTemplate = jdbcTemplate;
     }
 
-    @Override
-    public void init() {
-        Long maxId = jdbcTemplate.queryForObject(
-                "SELECT MAX(ID) FROM IMAGE",
-                Long.class
-        );
-        if (maxId == null) {
-            maxId = 0L;
-        }
-        counter.set(maxId);
-    }
-
     @Override
     public Metadata save(Metadata metadata) {
-        long id = counter.incrementAndGet();
         jdbcTemplate.update(
-                "INSERT INTO METADATA (ID, POSTED_BY, RESOLVED_NAMES, POST_ID_REF) VALUES (?,?,?,?)",
-                id,
+                "INSERT INTO METADATA (POST_ID_REF, POST_ID, POSTED_BY, RESOLVED_NAMES) VALUES (?,?,?,?)",
+                metadata.getPostIdRef(),
+                metadata.getPostId(),
                 metadata.getPostedBy(),
-                String.join("%sep%", metadata.getResolvedNames()),
-                metadata.getPostIdRef()
+                String.join("%sep%", metadata.getResolvedNames())
         );
-        metadata.setId(id);
-        applicationEventPublisher.publishEvent(new MetadataUpdateEvent(MetadataRepository.class, id, metadata.getPostIdRef()));
+        applicationEventPublisher.publishEvent(new MetadataUpdateEvent(MetadataRepository.class, metadata.getPostIdRef()));
         return metadata;
     }
 
-    @Override
-    public Optional<Metadata> findById(Long id) {
-        List<Metadata> metadata = jdbcTemplate.query(
-                "SELECT * FROM METADATA AS metadata WHERE metadata.ID = ?",
-                new Object[]{id},
-                new MetadataRowMapper()
-        );
-        if (metadata.isEmpty()) {
-            return Optional.empty();
-        } else {
-            return Optional.of(metadata.get(0));
-        }
-    }
-
     @Override
     public Optional<Metadata> findByPostId(String postId) {
         List<Metadata> metadata = jdbcTemplate.query(
-                "SELECT metadata.* FROM METADATA AS metadata INNER JOIN POST post ON post.ID = metadata.POST_ID_REF WHERE post.POST_ID = ?",
-                new Object[]{postId},
-                new MetadataRowMapper()
+                "SELECT metadata.* FROM METADATA AS metadata WHERE metadata.POST_ID = ?",
+                new MetadataRowMapper(),
+                postId
         );
         if (metadata.isEmpty()) {
             return Optional.empty();
@@ -83,7 +53,7 @@ public class MetadataRepository implements IMetadataRepository, ApplicationEvent
     @Override
     public int deleteByPostId(String postId) {
         return jdbcTemplate.update(
-                "DELETE FROM METADATA AS metadata WHERE metadata.ID = (SELECT inner_metadata.ID FROM POST AS post INNER JOIN METADATA inner_metadata ON post.ID = inner_metadata.POST_ID_REF WHERE post.POST_ID = ?)",
+                "DELETE FROM METADATA AS metadata WHERE metadata.POST_ID = ?",
                 postId
         );
     }

vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/MetadataRowMapper.java

@@ -12,13 +12,13 @@ public class MetadataRowMapper implements RowMapper<Metadata> {
     @Override
     public Metadata mapRow(ResultSet rs, int rowNum) throws SQLException {
         Metadata metadata = new Metadata();
-        metadata.setId(rs.getLong("ID"));
+        metadata.setPostIdRef(rs.getLong("POST_ID_REF"));
+        metadata.setPostId(rs.getString("POST_ID"));
         metadata.setPostedBy(rs.getString("POSTED_BY"));
         String resolvedNames = rs.getString("RESOLVED_NAMES");
         if (resolvedNames != null && !resolvedNames.isBlank()) {
             metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
         }
-        metadata.setPostIdRef(rs.getLong("POST_ID_REF"));
         return metadata;
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/PostRepository.java

@@ -4,6 +4,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationEventPublisher;
 import org.springframework.context.ApplicationEventPublisherAware;
 import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.lang.NonNull;
 import org.springframework.stereotype.Service;
 import tn.mnlr.vripper.event.PostRemoveEvent;
 import tn.mnlr.vripper.event.PostUpdateEvent;
@@ -13,13 +14,11 @@ import tn.mnlr.vripper.jpa.repositories.IPostRepository;
 
 import java.util.List;
 import java.util.Optional;
-import java.util.concurrent.atomic.AtomicLong;
 
 @Service
 public class PostRepository implements IPostRepository, ApplicationEventPublisherAware {
 
     private final JdbcTemplate jdbcTemplate;
-    private final AtomicLong counter = new AtomicLong(0);
     private ApplicationEventPublisher applicationEventPublisher;
 
     @Autowired
@@ -28,21 +27,15 @@ public class PostRepository implements IPostRepository, ApplicationEventPublishe
 
     }
 
-    @Override
-    public void init() {
-        Long maxId = jdbcTemplate.queryForObject(
-                "SELECT MAX(ID) FROM POST",
-                Long.class
-        );
-        if (maxId == null) {
-            maxId = 0L;
-        }
-        counter.set(maxId);
+    private synchronized Long nextId() {
+        return jdbcTemplate.queryForObject(
+                "CALL NEXT VALUE FOR SEQ_POST",
+                Long.class);
     }
 
     @Override
     public Post save(Post post) {
-        long id = counter.incrementAndGet();
+        long id = nextId();
         jdbcTemplate.update(
                 "INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
                 id,
@@ -69,7 +62,7 @@ public class PostRepository implements IPostRepository, ApplicationEventPublishe
     @Override
     public Optional<Post> findByPostId(String postId) {
         List<Post> posts = jdbcTemplate.query(
-                "SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE POST_ID = ?",
+                "SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE post.POST_ID = ?",
                 new PostRowMapper(),
                 postId);
         if (posts.isEmpty()) {
@@ -82,7 +75,7 @@ public class PostRepository implements IPostRepository, ApplicationEventPublishe
     @Override
     public List<String> findCompleted() {
         return jdbcTemplate.query(
-                "SELECT POST_ID FROM POST WHERE status = 'COMPLETE' AND done >= total",
+                "SELECT POST_ID FROM POST AS post WHERE status = 'COMPLETE' AND done >= total",
                 ((rs, rowNum) -> rs.getString("POST_ID"))
         );
     }
@@ -189,7 +182,7 @@ public class PostRepository implements IPostRepository, ApplicationEventPublishe
     }
 
     @Override
-    public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
+    public void setApplicationEventPublisher(@NonNull ApplicationEventPublisher applicationEventPublisher) {
         this.applicationEventPublisher = applicationEventPublisher;
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/PostRowMapper.java

@@ -19,34 +19,34 @@ public class PostRowMapper implements RowMapper<Post> {
 
         Post post = new Post();
         post.setId(rs.getLong("post.ID"));
-        post.setStatus(Status.valueOf(rs.getString("STATUS")));
-        post.setPostId(rs.getString("POST_ID"));
-        post.setThreadTitle(rs.getString("THREAD_TITLE"));
-        post.setThreadId(rs.getString("THREAD_ID"));
-        post.setTitle(rs.getString("TITLE"));
-        post.setUrl(rs.getString("URL"));
-        post.setDone(rs.getInt("DONE"));
-        post.setTotal(rs.getInt("TOTAL"));
-        post.setHosts(Set.of(rs.getString("HOSTS").split(DELIMITER)));
-        post.setForum(rs.getString("FORUM"));
-        post.setSecurityToken(rs.getString("SECURITY_TOKEN"));
-        post.setDownloadDirectory(rs.getString("POST_FOLDER_NAME"));
-        post.setThanked(rs.getBoolean("THANKED"));
+        post.setStatus(Status.valueOf(rs.getString("post.STATUS")));
+        post.setPostId(rs.getString("post.POST_ID"));
+        post.setThreadTitle(rs.getString("post.THREAD_TITLE"));
+        post.setThreadId(rs.getString("post.THREAD_ID"));
+        post.setTitle(rs.getString("post.TITLE"));
+        post.setUrl(rs.getString("post.URL"));
+        post.setDone(rs.getInt("post.DONE"));
+        post.setTotal(rs.getInt("post.TOTAL"));
+        post.setHosts(Set.of(rs.getString("post.HOSTS").split(DELIMITER)));
+        post.setForum(rs.getString("post.FORUM"));
+        post.setSecurityToken(rs.getString("post.SECURITY_TOKEN"));
+        post.setDownloadDirectory(rs.getString("post.POST_FOLDER_NAME"));
+        post.setThanked(rs.getBoolean("post.THANKED"));
         String previews;
-        if ((previews = rs.getString("PREVIEWS")) != null) {
+        if ((previews = rs.getString("post.PREVIEWS")) != null) {
             post.setPreviews(Set.of(previews.split(DELIMITER)));
         }
 
-        Long metadataId = rs.getLong("metadata.ID");
+        Long metadataId = rs.getLong("metadata.POST_ID_REF");
         if (!rs.wasNull()) {
             Metadata metadata = new Metadata();
-            metadata.setId(metadataId);
-            metadata.setPostIdRef(rs.getLong("POST_ID_REF"));
-            String resolvedNames = rs.getString("RESOLVED_NAMES");
+            metadata.setPostIdRef(metadataId);
+            metadata.setPostId(rs.getString("metadata.POST_ID"));
+            String resolvedNames = rs.getString("metadata.RESOLVED_NAMES");
             if (resolvedNames != null && !resolvedNames.isBlank()) {
                 metadata.setResolvedNames(List.of(resolvedNames.split("%sep%")));
             }
-            metadata.setPostedBy(rs.getString("POSTED_BY"));
+            metadata.setPostedBy(rs.getString("metadata.POSTED_BY"));
             post.setMetadata(metadata);
         }
 

vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/QueuedRepository.java

@@ -12,13 +12,11 @@ import tn.mnlr.vripper.jpa.repositories.IQueuedRepository;
 
 import java.util.List;
 import java.util.Optional;
-import java.util.concurrent.atomic.AtomicLong;
 
 @Service
 public class QueuedRepository implements IQueuedRepository, ApplicationEventPublisherAware {
 
     private final JdbcTemplate jdbcTemplate;
-    private final AtomicLong counter = new AtomicLong(0);
     private ApplicationEventPublisher applicationEventPublisher;
 
     @Autowired
@@ -26,21 +24,15 @@ public class QueuedRepository implements IQueuedRepository, ApplicationEventPubl
         this.jdbcTemplate = jdbcTemplate;
     }
 
-    @Override
-    public void init() {
-        Long maxId = jdbcTemplate.queryForObject(
-                "SELECT MAX(ID) FROM QUEUED",
-                Long.class
-        );
-        if (maxId == null) {
-            maxId = 0L;
-        }
-        counter.set(maxId);
+    private synchronized Long nextId() {
+        return jdbcTemplate.queryForObject(
+                "CALL NEXT VALUE FOR SEQ_QUEUED",
+                Long.class);
     }
 
     @Override
     public Queued save(Queued queued) {
-        long id = counter.incrementAndGet();
+        long id = nextId();
         jdbcTemplate.update(
                 "INSERT INTO QUEUED (ID, TOTAL, LINK, LOADING, POST_ID, THREAD_ID) values (?,?,?,?,?,?)",
                 id,
@@ -101,6 +93,11 @@ public class QueuedRepository implements IQueuedRepository, ApplicationEventPubl
         return mutationCount;
     }
 
+    @Override
+    public void deleteAll() {
+        jdbcTemplate.update("DELETE FROM QUEUED");
+    }
+
     @Override
     public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
         this.applicationEventPublisher = applicationEventPublisher;

vripper-server/src/main/java/tn/mnlr/vripper/listener/EventRemoveEventListener.java

@@ -0,0 +1,32 @@
+package tn.mnlr.vripper.listener;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.context.ApplicationListener;
+import org.springframework.stereotype.Component;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Sinks;
+import tn.mnlr.vripper.event.EventRemoveEvent;
+
+import javax.annotation.PreDestroy;
+
+@Component
+@Slf4j
+public class EventRemoveEventListener implements ApplicationListener<EventRemoveEvent>, DataEventListener<EventRemoveEvent> {
+
+    private final Sinks.Many<EventRemoveEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
+
+    @Override
+    public void onApplicationEvent(EventRemoveEvent event) {
+        sink.emitNext(event, EmitHandler.RETRY);
+    }
+
+    @Override
+    public Flux<EventRemoveEvent> getDataFlux() {
+        return sink.asFlux();
+    }
+
+    @PreDestroy
+    private void destroy() {
+        sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/listener/EventUpdateEventListener.java

@@ -0,0 +1,33 @@
+package tn.mnlr.vripper.listener;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.context.ApplicationListener;
+import org.springframework.lang.NonNull;
+import org.springframework.stereotype.Component;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Sinks;
+import tn.mnlr.vripper.event.EventUpdateEvent;
+
+import javax.annotation.PreDestroy;
+
+@Component
+@Slf4j
+public class EventUpdateEventListener implements ApplicationListener<EventUpdateEvent>, DataEventListener<EventUpdateEvent> {
+
+    private final Sinks.Many<EventUpdateEvent> sink = Sinks.many().multicast().onBackpressureBuffer();
+
+    @Override
+    public void onApplicationEvent(@NonNull EventUpdateEvent event) {
+        sink.emitNext(event, EmitHandler.RETRY);
+    }
+
+    @Override
+    public Flux<EventUpdateEvent> getDataFlux() {
+        return sink.asFlux();
+    }
+
+    @PreDestroy
+    private void destroy() {
+        sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST);
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/DataService.java

@@ -5,15 +5,13 @@ import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
-import tn.mnlr.vripper.jpa.domain.Image;
-import tn.mnlr.vripper.jpa.domain.Metadata;
-import tn.mnlr.vripper.jpa.domain.Post;
-import tn.mnlr.vripper.jpa.domain.Queued;
+import tn.mnlr.vripper.jpa.domain.*;
 import tn.mnlr.vripper.jpa.domain.enums.Status;
 import tn.mnlr.vripper.jpa.repositories.IImageRepository;
 import tn.mnlr.vripper.jpa.repositories.IMetadataRepository;
 import tn.mnlr.vripper.jpa.repositories.IPostRepository;
 import tn.mnlr.vripper.jpa.repositories.IQueuedRepository;
+import tn.mnlr.vripper.jpa.repositories.impl.EventRepository;
 
 import java.util.Collection;
 import java.util.List;
@@ -29,14 +27,16 @@ public class DataService {
     private final IQueuedRepository queuedRepository;
     private final IMetadataRepository metadataRepository;
     private final SettingsService settingsService;
+    private final EventRepository eventRepository;
 
     @Autowired
-    public DataService(IPostRepository postRepository, IImageRepository imageRepository, IQueuedRepository queuedRepository, IMetadataRepository metadataRepository, SettingsService settingsService) {
+    public DataService(IPostRepository postRepository, IImageRepository imageRepository, IQueuedRepository queuedRepository, IMetadataRepository metadataRepository, SettingsService settingsService, EventRepository eventRepository) {
         this.postRepository = postRepository;
         this.imageRepository = imageRepository;
         this.queuedRepository = queuedRepository;
         this.metadataRepository = metadataRepository;
         this.settingsService = settingsService;
+        this.eventRepository = eventRepository;
     }
 
     private void save(Post post) {
@@ -142,7 +142,7 @@ public class DataService {
         return imageRepository.findByPostId(postId);
     }
 
-    public Iterable<Post> findAllPosts() {
+    public List<Post> findAllPosts() {
         return postRepository.findAll();
     }
 
@@ -158,7 +158,7 @@ public class DataService {
         return queuedRepository.findByThreadId(threadId);
     }
 
-    public Iterable<Queued> findAllQueued() {
+    public List<Queued> findAllQueued() {
         return queuedRepository.findAll();
     }
 
@@ -174,9 +174,11 @@ public class DataService {
         return queuedRepository.findById(aLong);
     }
 
-    public void setMetadata(Post post, Metadata metadata) {
-        metadata.setPostIdRef(post.getId());
-        metadataRepository.save(metadata);
+    public synchronized void setMetadata(Post post, Metadata metadata) {
+        if (metadataRepository.findByPostId(post.getPostId()).isEmpty()) {
+            metadata.setPostIdRef(post.getId());
+            metadataRepository.save(metadata);
+        }
     }
 
     public Optional<Metadata> findMetadataByPostId(String postId) {
@@ -210,4 +212,16 @@ public class DataService {
     public void updatePostThanked(boolean thanked, Long id) {
         postRepository.updateThanked(thanked, id);
     }
+
+    public Optional<Event> findEventById(Long id) {
+        return eventRepository.findById(id);
+    }
+
+    public List<Event> findAllEvents() {
+        return eventRepository.findAll();
+    }
+
+    public void clearQueueLinks() {
+        queuedRepository.deleteAll();
+    }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/MetadataService.java

@@ -4,7 +4,6 @@ import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import lombok.Getter;
-import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 import net.jodah.failsafe.Failsafe;
 import org.apache.http.client.HttpClient;
@@ -12,18 +11,21 @@ import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.util.EntityUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.lang.NonNull;
 import org.springframework.stereotype.Service;
 import org.w3c.dom.Document;
 import org.w3c.dom.Node;
 import tn.mnlr.vripper.exception.DownloadException;
 import tn.mnlr.vripper.exception.PostParseException;
+import tn.mnlr.vripper.jpa.domain.Event;
 import tn.mnlr.vripper.jpa.domain.Metadata;
 import tn.mnlr.vripper.jpa.domain.Post;
+import tn.mnlr.vripper.jpa.repositories.IEventRepository;
+import tn.mnlr.vripper.services.domain.tasks.MetadataRunnable;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Objects;
+import java.time.LocalDateTime;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -39,12 +41,18 @@ public class MetadataService {
     private final VGAuthService VGAuthService;
     private final HtmlProcessorService htmlProcessorService;
     private final XpathService xpathService;
+    private final Map<String, MetadataRunnable> fetchingMetadata = new ConcurrentHashMap<>();
+    private final ThreadPoolService threadPoolService;
+    private final IEventRepository eventRepository;
+
     @Autowired
-    public MetadataService(ConnectionService cm, VGAuthService VGAuthService, HtmlProcessorService htmlProcessorService, XpathService xpathService) {
+    public MetadataService(ConnectionService cm, VGAuthService VGAuthService, HtmlProcessorService htmlProcessorService, XpathService xpathService, ThreadPoolService threadPoolService, IEventRepository eventRepository) {
         this.cm = cm;
         this.VGAuthService = VGAuthService;
         this.htmlProcessorService = htmlProcessorService;
         this.xpathService = xpathService;
+        this.threadPoolService = threadPoolService;
+        this.eventRepository = eventRepository;
 
         CacheLoader<Key, Metadata> loader = new CacheLoader<>() {
             @Override
@@ -57,14 +65,47 @@ public class MetadataService {
                 .build(loader);
     }
 
-    public Metadata get(Post post) throws ExecutionException {
+    public Metadata get(Post post) {
         Metadata metadata = new Metadata();
-        Metadata cachedMetadata = cache.get(new Key(post.getPostId(), post.getThreadId(), post.getUrl()));
+        Key key = new Key(post.getPostId(), post.getThreadId(), post.getUrl());
+        Metadata cachedMetadata = cache.getIfPresent(key);
+        if (cachedMetadata == null) {
+            Event event = new Event(Event.Type.METADATA_CACHE_MISS, Event.Status.PROCESSING, LocalDateTime.now(), "Loading metadata for " + post.getUrl());
+            eventRepository.save(event);
+            try {
+                cachedMetadata = cache.get(key);
+                event.setStatus(Event.Status.DONE);
+                eventRepository.update(event);
+            } catch (ExecutionException e) {
+                String error = "Failed to load metadata for " + post.getUrl();
+                log.error(error, e);
+                event.setStatus(Event.Status.ERROR);
+                event.setMessage(error + ": " + e.getMessage());
+                eventRepository.update(event);
+                return null;
+            }
+        }
         metadata.setPostedBy(cachedMetadata.getPostedBy());
+        metadata.setPostId(cachedMetadata.getPostId());
         metadata.setResolvedNames(List.copyOf(cachedMetadata.getResolvedNames()));
         return metadata;
     }
 
+    public void startFetchingMetadata(Post post) {
+        MetadataRunnable runnable = new MetadataRunnable(post);
+        threadPoolService.getGeneralExecutor().submit(runnable);
+        fetchingMetadata.put(post.getPostId(), runnable);
+    }
+
+    public void stopFetchingMetadata(Post post) {
+        this.fetchingMetadata.forEach((k, v) -> {
+            if (k.equals(post.getPostId())) {
+                v.setInterrupted(true);
+            }
+        });
+        fetchingMetadata.remove(post.getPostId());
+    }
+
     private Metadata fetchMetadata(Key key) {
         HttpGet httpGet = cm.buildHttpGet(key.getUrl(), null);
         Metadata metadata = new Metadata();
@@ -93,6 +134,8 @@ public class MetadataService {
 
                             Node node = xpathService.getAsNode(document, String.format("//div[@id='post_message_%s']", key.getPostId()));
                             metadata.setResolvedNames(findTitleInContent(node));
+
+                            metadata.setPostId(key.getPostId());
                         } catch (Exception e) {
                             throw new PostParseException(String.format("Failed to parse thread %s, post %s", key.getThreadId(), key.getPostId()), e);
                         } finally {

vripper-server/src/main/java/tn/mnlr/vripper/services/PostService.java

@@ -3,48 +3,43 @@ package tn.mnlr.vripper.services;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
-import lombok.Getter;
-import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.lang.NonNull;
 import org.springframework.stereotype.Service;
-import tn.mnlr.vripper.download.PendingQueue;
-import tn.mnlr.vripper.exception.PostParseException;
-import tn.mnlr.vripper.jpa.domain.Image;
+import tn.mnlr.vripper.jpa.domain.Event;
 import tn.mnlr.vripper.jpa.domain.Post;
 import tn.mnlr.vripper.jpa.domain.Queued;
-import tn.mnlr.vripper.jpa.domain.enums.Status;
-import tn.mnlr.vripper.services.domain.*;
+import tn.mnlr.vripper.jpa.repositories.IEventRepository;
+import tn.mnlr.vripper.services.domain.ApiThreadParser;
+import tn.mnlr.vripper.services.domain.MultiPostItem;
+import tn.mnlr.vripper.services.domain.tasks.AddPostRunnable;
+import tn.mnlr.vripper.services.domain.tasks.AddQueuedRunnable;
 
+import java.time.LocalDateTime;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
+import static tn.mnlr.vripper.jpa.domain.Event.Status.DONE;
+import static tn.mnlr.vripper.jpa.domain.Event.Status.PROCESSING;
+
 @Service
 @Slf4j
 public class PostService {
 
-    private final SettingsService settingsService;
-    private final PendingQueue pendingQueue;
     private final DataService dataService;
     private final ThreadPoolService threadPoolService;
-    private final Map<String, Future<?>> fetchingMetadata = new ConcurrentHashMap<>();
-    private final VGAuthService VGAuthService;
-
-    @Getter
+    private final MetadataService metadataService;
+    private final IEventRepository eventRepository;
     private final LoadingCache<Queued, List<MultiPostItem>> cache;
 
     @Autowired
-    public PostService(SettingsService settingsService, PendingQueue pendingQueue, DataService dataService, ThreadPoolService threadPoolService, VGAuthService VGAuthService) {
-        this.settingsService = settingsService;
-        this.pendingQueue = pendingQueue;
+    public PostService(DataService dataService, ThreadPoolService threadPoolService, MetadataService metadataService, IEventRepository eventRepository) {
         this.dataService = dataService;
         this.threadPoolService = threadPoolService;
-        this.VGAuthService = VGAuthService;
+        this.metadataService = metadataService;
+        this.eventRepository = eventRepository;
 
         CacheLoader<Queued, List<MultiPostItem>> loader = new CacheLoader<>() {
             @Override
@@ -59,95 +54,41 @@ public class PostService {
                 .build(loader);
     }
 
-    public void addPost(String postId, String threadId) throws PostParseException {
-
-        if (dataService.exists(postId)) {
-            log.warn(String.format("skipping %s, already loaded", postId));
-            return;
-        }
-
-        ApiPostParser apiPostParser = new ApiPostParser(threadId, postId);
-        ApiPost apiPost = apiPostParser.parse();
-        if (apiPost.getPost().isEmpty()) {
-            throw new PostParseException(String.format("parsing failed for thread %s, post %s", threadId, postId));
-        }
-
-        Post post = apiPost.getPost().get();
-        Set<Image> images = apiPost.getImages();
-
-        dataService.newPost(post, images);
-
-        // Metadata thread
-        fetchingMetadata.put(post.getPostId(), threadPoolService.getGeneralExecutor().submit(new MetadataRunnable(post)));
-
-        if (settingsService.getSettings().getAutoStart()) {
-            log.debug("Auto start downloads option is enabled");
-            post.setStatus(Status.PENDING);
-            try {
-                pendingQueue.enqueue(post, images);
-            } catch (InterruptedException e) {
-                log.warn("Interruption was caught");
-                Thread.currentThread().interrupt();
-                return;
-            }
-            log.debug(String.format("Done enqueuing jobs for %s", post.getUrl()));
-        } else {
-            post.setStatus(Status.STOPPED);
-            log.debug("Auto start downloads option is disabled");
-        }
-        if (!settingsService.getSettings().getLeaveThanksOnStart()) {
-            VGAuthService.leaveThanks(post);
-        }
-        dataService.updatePostStatus(post.getStatus(), post.getId());
-    }
-
     public void stopFetchingMetadata(Post post) {
-        this.fetchingMetadata.forEach((k, v) -> {
-            if (k.equals(post.getPostId())) {
-                v.cancel(true);
-            }
-        });
-        fetchingMetadata.remove(post.getPostId());
+        metadataService.stopFetchingMetadata(post);
     }
 
-    public void processMultiPost(List<Queued> queuedList) throws Exception {
+    public void processMultiPost(List<Queued> queuedList) {
         for (Queued queued : queuedList) {
             if (queued.getPostId() != null) {
-                addPost(queued.getPostId(), queued.getThreadId());
+                threadPoolService.getGeneralExecutor().submit(new AddPostRunnable(queued.getPostId(), queued.getThreadId()));
             } else {
-                threadPoolService.getGeneralExecutor().submit(() -> this.multiPost(queued));
+                threadPoolService.getGeneralExecutor().submit(new AddQueuedRunnable(queued));
             }
         }
     }
 
-    private void multiPost(Queued queued) {
-
+    public List<MultiPostItem> get(Queued queued) {
         List<MultiPostItem> multiPostItems;
-        try {
-            multiPostItems = cache.get(queued);
-        } catch (ExecutionException e) {
-            log.error(String.format("Failed to add post with thread id %s, postId %s", queued.getThreadId(), queued.getPostId()), e);
-            return;
-        }
-        queued.setTotal(multiPostItems.size());
-        queued.done();
-        log.debug(String.format("%d found for %s", multiPostItems.size(), queued.getLink()));
-        if (multiPostItems.size() == 1) {
+        multiPostItems = cache.getIfPresent(queued);
+        if (multiPostItems == null) {
+            Event event = new Event(Event.Type.QUEUED_CACHE_MISS, PROCESSING, LocalDateTime.now(), "Loading posts from " + queued.getLink());
+            eventRepository.save(event);
             try {
-                addPost(multiPostItems.get(0).getPostId(), multiPostItems.get(0).getThreadId());
-            } catch (PostParseException e) {
-                log.error(String.format("Failed to add post with postId %s", multiPostItems.get(0).getPostId()), e);
-                return;
-            }
-            log.debug(String.format("threadId %s, postId %s is added automatically for download", queued.getThreadId(), queued.getPostId()));
-        } else {
-            if (dataService.findQueuedByThreadId(queued.getThreadId()).isEmpty()) {
-                dataService.newQueueLink(queued);
-            } else {
-                log.info(String.format("Thread with id = %s is already loaded", queued.getThreadId()));
+                multiPostItems = cache.get(queued);
+                event.setStatus(DONE);
+                event.setMessage("Loaded " + multiPostItems.size() + " posts from " + queued.getLink());
+                eventRepository.update(event);
+            } catch (ExecutionException e) {
+                String error = String.format("Failed to parse link %s", queued.getLink());
+                log.error(error, e);
+                event.setStatus(Event.Status.ERROR);
+                event.setMessage(error + ": " + e.getMessage());
+                eventRepository.update(event);
+                return null;
             }
         }
-
+        return multiPostItems;
     }
 
     public void remove(String threadId) {

vripper-server/src/main/java/tn/mnlr/vripper/services/SettingsService.java

@@ -2,6 +2,7 @@ package tn.mnlr.vripper.services;
 
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.Getter;
@@ -10,6 +11,7 @@ import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.io.Resource;
 import org.springframework.stereotype.Service;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Sinks;
@@ -19,11 +21,17 @@ import tn.mnlr.vripper.listener.EmitHandler;
 
 import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
+import java.io.FileWriter;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.InvalidPathException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
 
 import static java.nio.file.StandardOpenOption.*;
 
@@ -33,28 +41,67 @@ import static java.nio.file.StandardOpenOption.*;
 public class SettingsService {
 
     private final Path configPath;
+    private final Path customProxiesPath;
     private final ObjectMapper om = new ObjectMapper();
 
+    private final Set<String> proxies = new HashSet<>();
+
     private Sinks.Many<Settings> sink = Sinks.many().multicast().onBackpressureBuffer();
 
     @Getter
     private Settings settings = new Settings();
+    @Value("classpath:proxies.json")
+    private Resource defaultProxies;
 
     public SettingsService(@Value("${base.dir}") String baseDir, @Value("${base.dir.name}") String baseDirName) {
         this.configPath = Paths.get(baseDir, baseDirName, "config.json");
+        this.customProxiesPath = Paths.get(baseDir, baseDirName, "proxies.json");
         om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
     }
 
     @PostConstruct
     private void init() {
+        loadViperProxies();
         restore();
         sink.emitNext(settings, EmitHandler.RETRY);
     }
 
+    private void loadViperProxies() {
+        try (InputStream defaultProxiesIs = defaultProxies.getInputStream()) {
+            List<String> defaultProxies = om.readValue(defaultProxiesIs, new TypeReference<>() {
+            });
+            List<String> customProxies = new ArrayList<>();
+            if (customProxiesPath.toFile().exists() && customProxiesPath.toFile().isFile()) {
+                customProxies = om.readValue(customProxiesPath.toFile(), new TypeReference<>() {
+                });
+            } else {
+                if (!customProxiesPath.toFile().createNewFile()) {
+                    log.warn("Unable to create " + customProxiesPath.toFile().getAbsolutePath());
+                } else {
+                    try (FileWriter fw = new FileWriter(customProxiesPath.toFile())) {
+                        fw.append("[]").append(System.lineSeparator());
+                        fw.flush();
+                    } catch (IOException e) {
+                        log.warn("Unable to create " + customProxiesPath.toFile().getAbsolutePath(), e);
+                    }
+                }
+            }
+            proxies.addAll(customProxies);
+            proxies.addAll(defaultProxies);
+        } catch (IOException e) {
+            log.error("Failed to load vipergirls proxies list", e);
+        }
+        proxies.add("https://vipergirls.to");
+    }
+
     Flux<Settings> getSettingsFlux() {
         return sink.asFlux();
     }
 
+    public List<String> getProxies() {
+        return new ArrayList<>(proxies);
+    }
+
     public void newSettings(Settings settings) {
 
         if (settings.getVLogin() != null && settings.getVLogin()) {
@@ -75,7 +122,9 @@ public class SettingsService {
 
     public void restore() {
         try {
-            settings = om.readValue(configPath.toFile(), Settings.class);
+            if (configPath.toFile().exists()) {
+                settings = om.readValue(configPath.toFile(), Settings.class);
+            }
         } catch (IOException e) {
             log.error("Failed restore user settings", e);
             settings = new Settings();
@@ -153,6 +202,14 @@ public class SettingsService {
             settings.setMaxAttempts(5);
         }
 
+        if (settings.getVProxy() == null || !proxies.contains(settings.getVProxy())) {
+            settings.setVProxy("https://vipergirls.to");
+        }
+
+        if (settings.getMaxEventLog() == null) {
+            settings.setMaxEventLog(1000);
+        }
+
         try {
             check(this.settings);
         } catch (ValidationException e) {
@@ -207,6 +264,10 @@ public class SettingsService {
         if (settings.getMaxAttempts() < 1 || settings.getMaxAttempts() > 10) {
             throw new ValidationException(String.format("Invalid maximum attempts settings, values must be in [%d,%d]", 1, 10));
         }
+
+        if (settings.getMaxEventLog() < 100 || settings.getMaxEventLog() > 10_000) {
+            throw new ValidationException(String.format("Invalid maximum event log record settings, values must be in [%d,%d]", 100, 10_000));
+        }
     }
 
     public Theme getTheme() {
@@ -289,5 +350,12 @@ public class SettingsService {
 
         @JsonProperty("maxAttempts")
         private Integer maxAttempts;
+
+        @JsonProperty("vProxy")
+        private String vProxy;
+
+        @JsonProperty("maxEventLog")
+        private Integer maxEventLog;
+
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/VGAuthService.java

@@ -20,6 +20,7 @@ import reactor.core.publisher.Sinks;
 import tn.mnlr.vripper.exception.VripperException;
 import tn.mnlr.vripper.jpa.domain.Post;
 import tn.mnlr.vripper.listener.EmitHandler;
+import tn.mnlr.vripper.services.domain.tasks.LeaveThanksRunnable;
 
 import javax.annotation.PostConstruct;
 import javax.annotation.PreDestroy;
@@ -33,7 +34,6 @@ public class VGAuthService {
     private final ConnectionService cm;
     private final SettingsService settingsService;
     private final ThreadPoolService threadPoolService;
-    private final DataService dataService;
 
     private final Disposable disposable;
 
@@ -46,11 +46,10 @@ public class VGAuthService {
     private String loggedUser = "";
 
     @Autowired
-    public VGAuthService(ConnectionService cm, SettingsService settingsService, ThreadPoolService threadPoolService, DataService dataService) {
+    public VGAuthService(ConnectionService cm, SettingsService settingsService, ThreadPoolService threadPoolService) {
         this.cm = cm;
         this.settingsService = settingsService;
         this.threadPoolService = threadPoolService;
-        this.dataService = dataService;
         disposable = settingsService.getSettingsFlux().subscribe(settings -> this.authenticate());
     }
 
@@ -94,7 +93,7 @@ public class VGAuthService {
             return;
         }
 
-        HttpPost postAuth = cm.buildHttpPost("https://vipergirls.to/login.php?do=login", null);
+        HttpPost postAuth = cm.buildHttpPost(settingsService.getSettings().getVProxy() + "/login.php?do=login", null);
         List<NameValuePair> params = new ArrayList<>();
         params.add(new BasicNameValuePair("vb_login_username", username));
 
@@ -107,12 +106,12 @@ public class VGAuthService {
             context.getCookieStore().clear();
             loggedUser = "";
             sink.emitNext(loggedUser, EmitHandler.RETRY);
-            log.error("Failed to authenticate user with vipergirls.to", e);
+            log.error("Failed to authenticate user with " + settingsService.getSettings().getVProxy(), e);
             return;
         }
 
-        postAuth.addHeader("Referer", "https://vipergirls.to/");
-        postAuth.addHeader("Host", "vipergirls.to");
+        postAuth.addHeader("Referer", settingsService.getSettings().getVProxy());
+        postAuth.addHeader("Host", settingsService.getSettings().getVProxy().replace("https://", "").replace("http://", ""));
 
         CloseableHttpClient client = cm.getClient().build();
 
@@ -125,14 +124,14 @@ public class VGAuthService {
             log.debug(String.format("Authentication with ViperGirls response body:%n%s", responseBody));
             EntityUtils.consumeQuietly(response.getEntity());
             if (context.getCookieStore().getCookies().stream().map(Cookie::getName).noneMatch(e -> e.equals("vg_userid"))) {
-                log.error("Failed to authenticate user with vipergirls.to, missing vg_userid cookie");
+                log.error(String.format("Failed to authenticate user with %s, missing vg_userid cookie", settingsService.getSettings().getVProxy()));
                 return;
             }
         } catch (Exception e) {
             context.getCookieStore().clear();
             loggedUser = "";
             sink.emitNext(loggedUser, EmitHandler.RETRY);
-            log.error("Failed to authenticate user with vipergirls.to", e);
+            log.error("Failed to authenticate user with " + settingsService.getSettings().getVProxy(), e);
             return;
         }
         authenticated = true;
@@ -142,62 +141,6 @@ public class VGAuthService {
     }
 
     public void leaveThanks(Post post) {
-        if (!settingsService.getSettings().getVLogin()) {
-            log.debug("Authentication with ViperGirls option is disabled");
-            return;
-        }
-        if (!settingsService.getSettings().getVThanks()) {
-            log.debug("Leave thanks option is disabled");
-            return;
-        }
-        if (!authenticated) {
-            log.error("You are not authenticated");
-            return;
-        }
-        if (post.isThanked()) {
-            log.debug("Already left a thanks");
-            return;
-        }
-        threadPoolService.getGeneralExecutor().submit(() -> {
-            try {
-                postThanks(post);
-            } catch (Exception e) {
-                log.error(String.format("Failed to leave a thanks for url %s, post id %s", post.getUrl(), post.getPostId()), e);
-            }
-        });
-    }
-
-    private void postThanks(Post post) throws VripperException {
-
-        HttpPost postThanks = cm.buildHttpPost("https://vipergirls.to/post_thanks.php", null);
-        List<NameValuePair> params = new ArrayList<>();
-        params.add(new BasicNameValuePair("do", "post_thanks_add"));
-        params.add(new BasicNameValuePair("using_ajax", "1"));
-        params.add(new BasicNameValuePair("p", post.getPostId()));
-        params.add(new BasicNameValuePair("securitytoken", post.getSecurityToken()));
-        try {
-            postThanks.setEntity(new UrlEncodedFormEntity(params));
-        } catch (Exception e) {
-            throw new VripperException(e);
-        }
-
-        postThanks.addHeader("Referer", "https://vipergirls.to/");
-        postThanks.addHeader("Host", "vipergirls.to");
-
-        CloseableHttpClient client = cm.getClient().build();
-
-        try (CloseableHttpResponse response = client.execute(postThanks, context)) {
-            if (response.getStatusLine().getStatusCode() / 100 == 2) {
-                post.setThanked(true);
-                dataService.updatePostThanked(post.isThanked(), post.getId());
-            }
-            EntityUtils.consumeQuietly(response.getEntity());
-        } catch (Exception e) {
-            throw new VripperException(e);
-        }
-
-        if (!post.isThanked()) {
-            throw new VripperException("Failed to leave");
-        }
+        threadPoolService.getGeneralExecutor().submit(new LeaveThanksRunnable(post, authenticated, context));
     }
 }

vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiPostHandler.java

@@ -7,6 +7,7 @@ import tn.mnlr.vripper.SpringContext;
 import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.jpa.domain.Image;
 import tn.mnlr.vripper.jpa.domain.Post;
+import tn.mnlr.vripper.services.SettingsService;
 
 import java.util.Collection;
 import java.util.HashSet;
@@ -25,7 +26,6 @@ class ApiPostHandler extends DefaultHandler {
     private final Set<Image> images = new HashSet<>();
     private Set<String> previews = new HashSet<>();
     private String threadTitle;
-    private String postTitle;
     private String forum;
     private String userHash;
     private int index = 0;
@@ -35,8 +35,9 @@ class ApiPostHandler extends DefaultHandler {
     ApiPostHandler(String threadId, String postId) {
         this.threadId = threadId;
         this.postId = postId;
-        this.postUrl = String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", this.threadId, this.postId, this.postId);
-        this.supportedHosts = SpringContext.getBeansOfType(Host.class).values();
+        supportedHosts = SpringContext.getBeansOfType(Host.class).values();
+        SettingsService settingsService = SpringContext.getBean(SettingsService.class);
+        postUrl = String.format("%s/threads/%s/?p=%s&viewfull=1#post%s", settingsService.getSettings().getVProxy(), this.threadId, this.postId, this.postId);
     }
 
 
@@ -62,7 +63,7 @@ class ApiPostHandler extends DefaultHandler {
                 threadTitle = attributes.getValue("title").trim();
                 break;
             case "post":
-                postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
+                String postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle);
                 parsedPost = new Post(
                         postTitle,
                         postUrl,

vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiPostParser.java

@@ -11,6 +11,7 @@ import tn.mnlr.vripper.SpringContext;
 import tn.mnlr.vripper.exception.DownloadException;
 import tn.mnlr.vripper.exception.PostParseException;
 import tn.mnlr.vripper.services.ConnectionService;
+import tn.mnlr.vripper.services.SettingsService;
 import tn.mnlr.vripper.services.VGAuthService;
 
 import javax.xml.parsers.SAXParserFactory;
@@ -20,20 +21,20 @@ import java.util.concurrent.atomic.AtomicReference;
 @Slf4j
 public class ApiPostParser {
 
-    private static final String VR_API = "https://vipergirls.to/vr.php";
-
     private static final SAXParserFactory factory = SAXParserFactory.newInstance();
 
     private final String threadId;
     private final String postId;
     private final ConnectionService cm;
     private final VGAuthService VGAuthService;
+    private final SettingsService settingsService;
 
     public ApiPostParser(String threadId, String postId) {
         this.threadId = threadId;
         this.postId = postId;
-        this.cm = SpringContext.getBean(ConnectionService.class);
-        this.VGAuthService = SpringContext.getBean(VGAuthService.class);
+        cm = SpringContext.getBean(ConnectionService.class);
+        VGAuthService = SpringContext.getBean(VGAuthService.class);
+        settingsService = SpringContext.getBean(SettingsService.class);
     }
 
     public ApiPost parse() throws PostParseException {
@@ -41,7 +42,7 @@ public class ApiPostParser {
         log.debug(String.format("Parsing post %s", postId));
         HttpGet httpGet;
         try {
-            URIBuilder uriBuilder = new URIBuilder(VR_API);
+            URIBuilder uriBuilder = new URIBuilder(settingsService.getSettings().getVProxy() + "/vr.php");
             uriBuilder.setParameter("p", postId);
             httpGet = cm.buildHttpGet(uriBuilder.build(), null);
         } catch (URISyntaxException e) {

vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiThreadHandler.java

@@ -6,6 +6,7 @@ import org.xml.sax.helpers.DefaultHandler;
 import tn.mnlr.vripper.SpringContext;
 import tn.mnlr.vripper.host.Host;
 import tn.mnlr.vripper.jpa.domain.Queued;
+import tn.mnlr.vripper.services.SettingsService;
 
 import java.util.*;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -15,6 +16,7 @@ public class ApiThreadHandler extends DefaultHandler {
 
     private final Queued queued;
     private final Collection<Host> supportedHosts;
+    private final SettingsService settingsService;
     private final Map<Host, AtomicInteger> hostMap = new HashMap<>();
     @Getter
     private final List<MultiPostItem> posts = new ArrayList<>();
@@ -28,7 +30,8 @@ public class ApiThreadHandler extends DefaultHandler {
 
     public ApiThreadHandler(Queued queued) {
         this.queued = queued;
-        this.supportedHosts = SpringContext.getBeansOfType(Host.class).values();
+        supportedHosts = SpringContext.getBeansOfType(Host.class).values();
+        settingsService = SpringContext.getBean(SettingsService.class);
     }
 
     @Override
@@ -70,7 +73,7 @@ public class ApiThreadHandler extends DefaultHandler {
                         postCounter,
                         postTitle,
                         imageCount,
-                        String.format("https://vipergirls.to/threads/?p=%s&viewfull=1#post%s", postId, postId),
+                        String.format("%s/threads/?p=%s&viewfull=1#post%s", settingsService.getSettings().getVProxy(), postId, postId),
                         previews,
                         hostMap.entrySet().stream().filter(v -> v.getValue().get() > 0).map(e -> e.getKey().getHost() + " (" + e.getValue().get() + ")").collect(Collectors.joining(", "))
                 ));

vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiThreadParser.java

@@ -12,6 +12,7 @@ import tn.mnlr.vripper.exception.DownloadException;
 import tn.mnlr.vripper.exception.PostParseException;
 import tn.mnlr.vripper.jpa.domain.Queued;
 import tn.mnlr.vripper.services.ConnectionService;
+import tn.mnlr.vripper.services.SettingsService;
 import tn.mnlr.vripper.services.VGAuthService;
 
 import javax.xml.parsers.SAXParserFactory;
@@ -23,17 +24,17 @@ import java.util.concurrent.atomic.AtomicReference;
 @Slf4j
 public class ApiThreadParser {
 
-    private static final String VR_API = "https://vipergirls.to/vr.php";
-
     private static final SAXParserFactory factory = SAXParserFactory.newInstance();
     private final Queued queued;
     private final ConnectionService cm;
     private final VGAuthService VGAuthService;
+    private final SettingsService settingsService;
 
     public ApiThreadParser(Queued queued) {
         this.queued = queued;
-        this.cm = SpringContext.getBean(ConnectionService.class);
-        this.VGAuthService = SpringContext.getBean(VGAuthService.class);
+        cm = SpringContext.getBean(ConnectionService.class);
+        VGAuthService = SpringContext.getBean(VGAuthService.class);
+        settingsService = SpringContext.getBean(SettingsService.class);
     }
 
     public List<MultiPostItem> parse() throws PostParseException {
@@ -41,7 +42,7 @@ public class ApiThreadParser {
         log.debug(String.format("Parsing thread %s", queued));
         HttpGet httpGet;
         try {
-            URIBuilder uriBuilder = new URIBuilder(VR_API);
+            URIBuilder uriBuilder = new URIBuilder(settingsService.getSettings().getVProxy() + "/vr.php");
             uriBuilder.setParameter("t", queued.getThreadId());
             httpGet = cm.buildHttpGet(uriBuilder.build(), null);
         } catch (URISyntaxException e) {

vripper-server/src/main/java/tn/mnlr/vripper/services/domain/MetadataRunnable.java

@@ -1,45 +0,0 @@
-package tn.mnlr.vripper.services.domain;
-
-import lombok.Getter;
-import lombok.NonNull;
-import lombok.extern.slf4j.Slf4j;
-import net.jodah.failsafe.FailsafeException;
-import org.apache.http.impl.execchain.RequestAbortedException;
-import tn.mnlr.vripper.SpringContext;
-import tn.mnlr.vripper.jpa.domain.Metadata;
-import tn.mnlr.vripper.jpa.domain.Post;
-import tn.mnlr.vripper.services.DataService;
-import tn.mnlr.vripper.services.MetadataService;
-
-@Slf4j
-public class MetadataRunnable implements Runnable {
-
-    @Getter
-    private final Post post;
-
-    private final MetadataService metadataService;
-    private final DataService dataService;
-
-    public MetadataRunnable(@NonNull Post post) {
-        this.post = post;
-        this.metadataService = SpringContext.getBean(MetadataService.class);
-        this.dataService = SpringContext.getBean(DataService.class);
-    }
-
-    @Override
-    public void run() {
-        try {
-            if (Thread.interrupted()) {
-                log.debug(String.format("Metadata fetching for postId=%s, threadId=%s interrupted", post.getPostId(), post.getThreadId()));
-                return;
-            }
-            Metadata metadata = metadataService.get(post);
-            dataService.setMetadata(post, metadata);
-        } catch (Exception e) {
-            if (e.getCause() instanceof InterruptedException || (e.getCause() instanceof FailsafeException && (e.getCause().getCause() instanceof InterruptedException || e.getCause().getCause() instanceof RequestAbortedException))) {
-                return;
-            }
-            log.error(String.format("Failed to get metadata for postId %s", post.getPostId()), e);
-        }
-    }
-}

vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddPostRunnable.java

@@ -0,0 +1,139 @@
+package tn.mnlr.vripper.services.domain.tasks;
+
+import lombok.extern.slf4j.Slf4j;
+import tn.mnlr.vripper.SpringContext;
+import tn.mnlr.vripper.Utils;
+import tn.mnlr.vripper.download.PendingQueue;
+import tn.mnlr.vripper.exception.PostParseException;
+import tn.mnlr.vripper.jpa.domain.Event;
+import tn.mnlr.vripper.jpa.domain.Image;
+import tn.mnlr.vripper.jpa.domain.Post;
+import tn.mnlr.vripper.jpa.domain.enums.Status;
+import tn.mnlr.vripper.jpa.repositories.IEventRepository;
+import tn.mnlr.vripper.services.DataService;
+import tn.mnlr.vripper.services.MetadataService;
+import tn.mnlr.vripper.services.SettingsService;
+import tn.mnlr.vripper.services.VGAuthService;
+import tn.mnlr.vripper.services.domain.ApiPost;
+import tn.mnlr.vripper.services.domain.ApiPostParser;
+
+import java.time.LocalDateTime;
+import java.util.Set;
+
+import static tn.mnlr.vripper.jpa.domain.Event.Status.ERROR;
+import static tn.mnlr.vripper.jpa.domain.Event.Status.PROCESSING;
+
+@Slf4j
+public class AddPostRunnable implements Runnable {
+
+    private final String postId;
+    private final String threadId;
+    private final DataService dataService;
+    private final MetadataService metadataService;
+    private final SettingsService settingsService;
+    private final PendingQueue pendingQueue;
+    private final VGAuthService VGAuthService;
+    private final Event event;
+    private final IEventRepository eventRepository;
+    private final String link;
+
+    public AddPostRunnable(String postId, String threadId) {
+        this.postId = postId;
+        this.threadId = threadId;
+        this.dataService = SpringContext.getBean(DataService.class);
+        this.metadataService = SpringContext.getBean(MetadataService.class);
+        this.settingsService = SpringContext.getBean(SettingsService.class);
+        this.pendingQueue = SpringContext.getBean(PendingQueue.class);
+        this.VGAuthService = SpringContext.getBean(VGAuthService.class);
+        this.eventRepository = SpringContext.getBean(IEventRepository.class);
+        link = settingsService.getSettings().getVProxy() + String.format("/%s?%s", threadId, (postId != null ? "p=" + postId : ""));
+        event = new Event(Event.Type.POST, Event.Status.PENDING, LocalDateTime.now(), String.format("Processing %s", link));
+        eventRepository.save(event);
+    }
+
+    @Override
+    public void run() {
+
+        try {
+            event.setStatus(PROCESSING);
+            eventRepository.update(event);
+            if (dataService.exists(postId)) {
+                log.warn(String.format("skipping %s, already loaded", postId));
+                event.setMessage(String.format("Gallery %s is already loaded", link));
+                event.setStatus(ERROR);
+                eventRepository.update(event);
+                return;
+            }
+
+            ApiPostParser apiPostParser = new ApiPostParser(threadId, postId);
+
+            ApiPost apiPost;
+            try {
+                apiPost = apiPostParser.parse();
+            } catch (PostParseException e) {
+                String error = String.format("parsing failed for gallery %s", link);
+                log.error(error, e);
+                event.setMessage(error + "\n" + Utils.throwableToString(e));
+                event.setStatus(ERROR);
+                eventRepository.update(event);
+                return;
+            }
+            if (apiPost.getPost().isEmpty()) {
+                String error = String.format("Gallery %s contains no galleries", link);
+                log.error(error);
+                event.setMessage(error);
+                event.setStatus(ERROR);
+                eventRepository.update(event);
+                return;
+            }
+            if (apiPost.getImages().isEmpty()) {
+                String error = String.format("Gallery %s contains no images to download", link);
+                log.error(error);
+                event.setMessage(error);
+                event.setStatus(ERROR);
+                eventRepository.update(event);
+                return;
+            }
+
+            Post post = apiPost.getPost().get();
+            Set<Image> images = apiPost.getImages();
+
+            dataService.newPost(post, images);
+
+            metadataService.startFetchingMetadata(post);
+
+            if (settingsService.getSettings().getAutoStart()) {
+                log.debug("Auto start downloads option is enabled");
+                post.setStatus(Status.PENDING);
+                try {
+                    pendingQueue.enqueue(post, images);
+                } catch (InterruptedException e) {
+                    log.warn("Interruption was caught");
+                    Thread.currentThread().interrupt();
+                    return;
+                }
+                log.debug(String.format("Done enqueuing jobs for %s", post.getUrl()));
+            } else {
+                post.setStatus(Status.STOPPED);
+                log.debug("Auto start downloads option is disabled");
+            }
+            if (!settingsService.getSettings().getLeaveThanksOnStart()) {
+                VGAuthService.leaveThanks(post);
+            }
+            dataService.updatePostStatus(post.getStatus(), post.getId());
+            event.setMessage(String.format("Gallery %s is successfully added to download queue", link));
+            event.setStatus(Event.Status.DONE);
+            eventRepository.update(event);
+        } catch (Exception e) {
+            try {
+                String error = String.format("Error when adding gallery %s", link);
+                log.error(error, e);
+                event.setMessage(error + "\n" + Utils.throwableToString(e));
+                event.setStatus(ERROR);
+                eventRepository.update(event);
+            } catch (Exception exp) {
+                log.error(exp.getMessage(), exp);
+            }
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddQueuedRunnable.java

@@ -0,0 +1,83 @@
+package tn.mnlr.vripper.services.domain.tasks;
+
+import lombok.extern.slf4j.Slf4j;
+import tn.mnlr.vripper.SpringContext;
+import tn.mnlr.vripper.Utils;
+import tn.mnlr.vripper.jpa.domain.Event;
+import tn.mnlr.vripper.jpa.domain.Queued;
+import tn.mnlr.vripper.jpa.repositories.IEventRepository;
+import tn.mnlr.vripper.services.DataService;
+import tn.mnlr.vripper.services.PostService;
+import tn.mnlr.vripper.services.ThreadPoolService;
+import tn.mnlr.vripper.services.domain.MultiPostItem;
+
+import java.time.LocalDateTime;
+import java.util.List;
+
+import static tn.mnlr.vripper.jpa.domain.Event.Status.ERROR;
+import static tn.mnlr.vripper.jpa.domain.Event.Status.PROCESSING;
+
+@Slf4j
+public class AddQueuedRunnable implements Runnable {
+
+    private final Queued queued;
+    private final ThreadPoolService threadPoolService;
+    private final DataService dataService;
+    private final IEventRepository eventRepository;
+    private final PostService postService;
+    private final Event event;
+
+    public AddQueuedRunnable(Queued queued) {
+        this.queued = queued;
+        threadPoolService = SpringContext.getBean(ThreadPoolService.class);
+        dataService = SpringContext.getBean(DataService.class);
+        eventRepository = SpringContext.getBean(IEventRepository.class);
+        postService = SpringContext.getBean(PostService.class);
+        event = new Event(Event.Type.QUEUED, Event.Status.PENDING, LocalDateTime.now(), String.format("Processing multi-post link %s", queued.getLink()));
+        eventRepository.save(event);
+    }
+
+    @Override
+    public void run() {
+        try {
+            event.setStatus(PROCESSING);
+            eventRepository.update(event);
+            List<MultiPostItem> multiPostItems = postService.get(queued);
+            if (multiPostItems == null) {
+                String message = String.format("Fetching multi-post link %s failed", queued.getLink());
+                event.setStatus(ERROR);
+                event.setMessage(message);
+                eventRepository.update(event);
+                return;
+            }
+            queued.setTotal(multiPostItems.size());
+            queued.done();
+            if (multiPostItems.size() == 1) {
+                threadPoolService.getGeneralExecutor().submit(new AddPostRunnable(multiPostItems.get(0).getPostId(), multiPostItems.get(0).getThreadId()));
+                event.setStatus(Event.Status.DONE);
+                event.setMessage(String.format("Link %s is added to download queue", queued.getLink()));
+            } else {
+                if (dataService.findQueuedByThreadId(queued.getThreadId()).isEmpty()) {
+                    dataService.newQueueLink(queued);
+                    event.setStatus(Event.Status.DONE);
+                    event.setMessage(String.format("Link %s is added to multi-post links", queued.getLink()));
+                } else {
+                    log.info(String.format("Link %s is already loaded", queued.getLink()));
+                    event.setStatus(Event.Status.ERROR);
+                    event.setMessage(String.format("%s has already been added to multi-post links", queued.getLink()));
+                }
+            }
+            eventRepository.update(event);
+        } catch (Exception e) {
+            try {
+                String error = String.format("Error when adding multi-post link %s", queued.getLink());
+                log.error(error, e);
+                event.setMessage(error + "\n" + Utils.throwableToString(e));
+                event.setStatus(ERROR);
+                eventRepository.update(event);
+            } catch (Exception exp) {
+                log.error(exp.getMessage(), exp);
+            }
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LeaveThanksRunnable.java

@@ -0,0 +1,123 @@
+package tn.mnlr.vripper.services.domain.tasks;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
+import tn.mnlr.vripper.SpringContext;
+import tn.mnlr.vripper.Utils;
+import tn.mnlr.vripper.jpa.domain.Event;
+import tn.mnlr.vripper.jpa.domain.Post;
+import tn.mnlr.vripper.jpa.repositories.IEventRepository;
+import tn.mnlr.vripper.services.ConnectionService;
+import tn.mnlr.vripper.services.DataService;
+import tn.mnlr.vripper.services.SettingsService;
+
+import java.io.UnsupportedEncodingException;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
+
+@Slf4j
+public class LeaveThanksRunnable implements Runnable {
+
+    private final ConnectionService cm;
+    private final DataService dataService;
+    private final HttpClientContext context;
+    private final SettingsService settingsService;
+    private final IEventRepository eventRepository;
+    private final boolean authenticated;
+    private final Post post;
+    private final Event event;
+
+    public LeaveThanksRunnable(Post post, boolean authenticated, HttpClientContext context) {
+        this.post = post;
+        this.authenticated = authenticated;
+        this.context = context;
+        cm = SpringContext.getBean(ConnectionService.class);
+        dataService = SpringContext.getBean(DataService.class);
+        eventRepository = SpringContext.getBean(IEventRepository.class);
+        settingsService = SpringContext.getBean(SettingsService.class);
+        event = new Event(Event.Type.THANKS, Event.Status.PENDING, LocalDateTime.now(), String.format("Leaving thanks for %s", post.getUrl()));
+        eventRepository.save(event);
+    }
+
+    @Override
+    public void run() {
+        try {
+            event.setStatus(Event.Status.PROCESSING);
+            eventRepository.update(event);
+            if (!settingsService.getSettings().getVLogin()) {
+                event.setMessage(String.format("Will not leave a thanks for %s\nAuthentication with ViperGirls option is disabled", post.getUrl()));
+                event.setStatus(Event.Status.DONE);
+                eventRepository.update(event);
+                return;
+            }
+            if (!settingsService.getSettings().getVThanks()) {
+                event.setMessage(String.format("Will not leave a thanks for %s\nLeave thanks option is disabled", post.getUrl()));
+                event.setStatus(Event.Status.DONE);
+                eventRepository.update(event);
+                return;
+            }
+            if (!authenticated) {
+                event.setMessage(String.format("Will not leave a thanks for %s\nYou are not authenticated", post.getUrl()));
+                event.setStatus(Event.Status.ERROR);
+                eventRepository.update(event);
+                return;
+            }
+            if (post.isThanked()) {
+                event.setMessage(String.format("Will not leave a thanks for %s\nAlready left a thanks", post.getUrl()));
+                event.setStatus(Event.Status.DONE);
+                eventRepository.update(event);
+                return;
+            }
+
+            HttpPost postThanks = cm.buildHttpPost(settingsService.getSettings().getVProxy() + "/post_thanks.php", null);
+            List<NameValuePair> params = new ArrayList<>();
+            params.add(new BasicNameValuePair("do", "post_thanks_add"));
+            params.add(new BasicNameValuePair("using_ajax", "1"));
+            params.add(new BasicNameValuePair("p", post.getPostId()));
+            params.add(new BasicNameValuePair("securitytoken", post.getSecurityToken()));
+            try {
+                postThanks.setEntity(new UrlEncodedFormEntity(params));
+            } catch (UnsupportedEncodingException e) {
+                String error = String.format("Request error for %s", post.getUrl());
+                log.error(error, e);
+                event.setMessage(error + "\n" + Utils.throwableToString(e));
+                event.setStatus(Event.Status.ERROR);
+                eventRepository.update(event);
+                return;
+            }
+
+            postThanks.addHeader("Referer", settingsService.getSettings().getVProxy());
+            postThanks.addHeader("Host", settingsService.getSettings().getVProxy().replace("https://", "").replace("http://", ""));
+
+            CloseableHttpClient client = cm.getClient().build();
+
+            try (CloseableHttpResponse response = client.execute(postThanks, context)) {
+                if (response.getStatusLine().getStatusCode() / 100 == 2) {
+                    post.setThanked(true);
+                    dataService.updatePostThanked(post.isThanked(), post.getId());
+                }
+                EntityUtils.consumeQuietly(response.getEntity());
+            }
+            event.setStatus(Event.Status.DONE);
+            eventRepository.update(event);
+        } catch (Exception e) {
+            try {
+                String error = String.format("Failed to leave a thanks for %s", post.getUrl());
+                log.error(error, e);
+                event.setMessage(error + "\n" + Utils.throwableToString(e));
+                event.setStatus(Event.Status.ERROR);
+                eventRepository.update(event);
+            } catch (Exception exp) {
+                log.error(exp.getMessage(), exp);
+            }
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LinkScanRunnable.java

@@ -0,0 +1,99 @@
+package tn.mnlr.vripper.services.domain.tasks;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.lang.NonNull;
+import tn.mnlr.vripper.SpringContext;
+import tn.mnlr.vripper.Utils;
+import tn.mnlr.vripper.jpa.domain.Event;
+import tn.mnlr.vripper.jpa.domain.Queued;
+import tn.mnlr.vripper.jpa.repositories.impl.EventRepository;
+import tn.mnlr.vripper.services.PostService;
+import tn.mnlr.vripper.services.SettingsService;
+
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static tn.mnlr.vripper.jpa.domain.Event.Status.ERROR;
+
+@Slf4j
+public class LinkScanRunnable implements Runnable {
+
+    private final static Object LOCK = new Object();
+    private final List<String> urlList;
+    private final SettingsService settingsService;
+    private final PostService postService;
+    private final EventRepository eventRepository;
+    private final Event event;
+
+
+    public LinkScanRunnable(@NonNull List<String> urlList) {
+        this.urlList = urlList;
+        settingsService = SpringContext.getBean(SettingsService.class);
+        postService = SpringContext.getBean(PostService.class);
+        eventRepository = SpringContext.getBean(EventRepository.class);
+        event = new Event(Event.Type.SCAN, Event.Status.PENDING, LocalDateTime.now(), "Links scan:\n\t" + String.join("\n\t", urlList));
+        eventRepository.save(event);
+    }
+
+    @Override
+    public void run() {
+        synchronized (LOCK) {
+            try {
+                event.setStatus(Event.Status.PROCESSING);
+                eventRepository.update(event);
+                ArrayList<Queued> queuedList = new ArrayList<>();
+                List<String> unsupported = new ArrayList<>();
+                List<String> unrecognized = new ArrayList<>();
+                for (String url : urlList) {
+                    log.debug(String.format("Starting to process thread: %s", url));
+                    if (!url.startsWith(settingsService.getSettings().getVProxy())) {
+                        log.error(String.format("Unsupported link %s", url));
+                        unsupported.add(url);
+                        continue;
+                    }
+
+                    String threadId, postId;
+                    Matcher m = Pattern.compile(Pattern.quote(settingsService.getSettings().getVProxy()) + "/threads/(\\d+)((.*p=)(\\d+))?").matcher(url);
+                    if (m.find()) {
+                        threadId = m.group(1);
+                        postId = m.group(4);
+                    } else {
+                        log.error(String.format("Cannot retrieve thread id from URL %s", url));
+                        unrecognized.add(url);
+                        continue;
+                    }
+                    queuedList.add(new Queued(url, threadId, postId));
+                }
+                StringBuilder errorMessage = new StringBuilder();
+                if (!unsupported.isEmpty()) {
+                    errorMessage.append("Unsupported links:\n\t").append(String.join("\n\t", unsupported)).append("\n\n");
+                }
+                if (!unrecognized.isEmpty()) {
+                    errorMessage.append("Unrecognized links:\n\t").append(String.join("\n\t", unrecognized)).append("\n\n");
+                }
+
+                postService.processMultiPost(queuedList);
+                if (!unsupported.isEmpty() || !unrecognized.isEmpty()) {
+                    event.setStatus(Event.Status.ERROR);
+                    event.setMessage("Some links failed to be scanned: \n" + errorMessage.toString());
+                } else {
+                    event.setStatus(Event.Status.DONE);
+                }
+                eventRepository.update(event);
+            } catch (Exception e) {
+                try {
+                    String error = "Error when scanning links";
+                    log.error(error, e);
+                    event.setMessage(error + "\n" + Utils.throwableToString(e));
+                    event.setStatus(ERROR);
+                    eventRepository.update(event);
+                } catch (Exception exp) {
+                    log.error(exp.getMessage(), exp);
+                }
+            }
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/MetadataRunnable.java

@@ -0,0 +1,82 @@
+package tn.mnlr.vripper.services.domain.tasks;
+
+import lombok.Getter;
+import lombok.NonNull;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+import tn.mnlr.vripper.SpringContext;
+import tn.mnlr.vripper.Utils;
+import tn.mnlr.vripper.jpa.domain.Event;
+import tn.mnlr.vripper.jpa.domain.Metadata;
+import tn.mnlr.vripper.jpa.domain.Post;
+import tn.mnlr.vripper.jpa.repositories.IEventRepository;
+import tn.mnlr.vripper.services.DataService;
+import tn.mnlr.vripper.services.MetadataService;
+
+import java.time.LocalDateTime;
+
+import static tn.mnlr.vripper.jpa.domain.Event.Status.ERROR;
+
+@Slf4j
+public class MetadataRunnable implements Runnable {
+
+    @Getter
+    private final Post post;
+
+    private final MetadataService metadataService;
+    private final DataService dataService;
+    private final IEventRepository eventRepository;
+
+    private final Event event;
+
+    @Setter
+    private volatile boolean interrupted = false;
+
+    public MetadataRunnable(@NonNull Post post) {
+        this.post = post;
+        metadataService = SpringContext.getBean(MetadataService.class);
+        dataService = SpringContext.getBean(DataService.class);
+        eventRepository = SpringContext.getBean(IEventRepository.class);
+        event = new Event(Event.Type.METADATA, Event.Status.PENDING, LocalDateTime.now(), "Fetching metadata for " + post.getUrl());
+        eventRepository.save(event);
+    }
+
+    @Override
+    public void run() {
+
+        try {
+            event.setStatus(Event.Status.PROCESSING);
+            eventRepository.update(event);
+            if (interrupted) {
+                String message = String.format("Fetching metadata for %s interrupted", post.getUrl());
+                event.setStatus(Event.Status.DONE);
+                event.setMessage(message);
+                eventRepository.update(event);
+                return;
+            }
+
+            Metadata metadata = metadataService.get(post);
+            if (metadata == null) {
+                String message = String.format("Fetching metadata for %s failed", post.getUrl());
+                event.setStatus(ERROR);
+                event.setMessage(message);
+                eventRepository.update(event);
+                return;
+            }
+            dataService.setMetadata(post, metadata);
+
+            event.setStatus(Event.Status.DONE);
+            eventRepository.update(event);
+        } catch (Exception e) {
+            try {
+                String message = String.format("Failed to fetch metadata for %s", post.getUrl());
+                log.error(message, e);
+                event.setMessage(message + "\n" + Utils.throwableToString(e));
+                event.setStatus(ERROR);
+                eventRepository.update(event);
+            } catch (Exception exp) {
+                log.error(exp.getMessage(), exp);
+            }
+        }
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/EventLogRestEndpoint.java

@@ -0,0 +1,27 @@
+package tn.mnlr.vripper.web.restendpoints;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.CrossOrigin;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.ResponseStatus;
+import org.springframework.web.bind.annotation.RestController;
+import tn.mnlr.vripper.jpa.repositories.IEventRepository;
+
+@Slf4j
+@RestController
+@CrossOrigin(value = "*")
+public class EventLogRestEndpoint {
+
+    private final IEventRepository eventRepository;
+
+    public EventLogRestEndpoint(IEventRepository eventRepository) {
+        this.eventRepository = eventRepository;
+    }
+
+    @GetMapping("/events/clear")
+    @ResponseStatus(value = HttpStatus.OK)
+    public void clear() {
+        eventRepository.deleteAll();
+    }
+}

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java

@@ -6,15 +6,13 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
 import org.springframework.web.bind.annotation.*;
 import tn.mnlr.vripper.download.DownloadService;
-import tn.mnlr.vripper.exception.PostParseException;
 import tn.mnlr.vripper.jpa.domain.Metadata;
 import tn.mnlr.vripper.jpa.domain.Post;
 import tn.mnlr.vripper.jpa.domain.Queued;
-import tn.mnlr.vripper.services.DataService;
-import tn.mnlr.vripper.services.PathService;
-import tn.mnlr.vripper.services.PostService;
-import tn.mnlr.vripper.services.ThreadPoolService;
+import tn.mnlr.vripper.services.*;
 import tn.mnlr.vripper.services.domain.MultiPostItem;
+import tn.mnlr.vripper.services.domain.tasks.AddPostRunnable;
+import tn.mnlr.vripper.services.domain.tasks.LinkScanRunnable;
 import tn.mnlr.vripper.web.restendpoints.domain.*;
 import tn.mnlr.vripper.web.restendpoints.exceptions.BadRequestException;
 import tn.mnlr.vripper.web.restendpoints.exceptions.NotFoundException;
@@ -24,9 +22,6 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
-import java.util.concurrent.ExecutionException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 @Slf4j
@@ -34,21 +29,22 @@ import java.util.stream.Collectors;
 @CrossOrigin(value = "*")
 public class PostRestEndpoint {
 
-    private static final Pattern VG_URL_PATTERN = Pattern.compile("https://vipergirls\\.to/threads/(\\d+)((.*p=)(\\d+))?");
     private static final Object LOCK = new Object();
     private final DataService dataService;
     private final PathService pathService;
     private final DownloadService downloadService;
     private final PostService postService;
     private final ThreadPoolService threadPoolService;
+    private final SettingsService settingsService;
 
     @Autowired
-    public PostRestEndpoint(DataService dataService, PathService pathService, DownloadService downloadService, PostService postService, ThreadPoolService threadPoolService) {
+    public PostRestEndpoint(DataService dataService, PathService pathService, DownloadService downloadService, PostService postService, ThreadPoolService threadPoolService, SettingsService settingsService) {
         this.dataService = dataService;
         this.pathService = pathService;
         this.downloadService = downloadService;
         this.postService = postService;
         this.threadPoolService = threadPoolService;
+        this.settingsService = settingsService;
     }
 
     @PostMapping("/post")
@@ -60,30 +56,7 @@ public class PostRestEndpoint {
                 throw new BadRequestException("Cannot process empty requests");
             }
             List<String> urlList = Arrays.stream(_url.getUrl().split("\\r?\\n")).map(String::trim).filter(e -> !e.isEmpty()).collect(Collectors.toList());
-            ArrayList<Queued> queuedList = new ArrayList<>();
-            for (String url : urlList) {
-                log.debug(String.format("Starting to process thread: %s", url));
-                if (!url.startsWith("https://vipergirls.to")) {
-                    log.error(String.format("Unsupported link %s", url));
-                    continue;
-                }
-
-                String threadId, postId;
-                Matcher m = VG_URL_PATTERN.matcher(url);
-                if (m.find()) {
-                    threadId = m.group(1);
-                    postId = m.group(4);
-                } else {
-                    throw new BadRequestException(String.format("Cannot retrieve thread id from URL %s", url));
-                }
-                queuedList.add(new Queued(url, threadId, postId));
-            }
-            try {
-                postService.processMultiPost(queuedList);
-            } catch (Exception e) {
-                log.error("Failed to parse links", e);
-                throw new ServerErrorException(e.getMessage());
-            }
+            threadPoolService.getGeneralExecutor().submit(new LinkScanRunnable(urlList));
         }
     }
 
@@ -100,14 +73,7 @@ public class PostRestEndpoint {
     public void addPost(@RequestBody List<PostToAdd> posts) {
         synchronized (LOCK) {
             for (PostToAdd post : posts) {
-                threadPoolService.getGeneralExecutor().submit(() -> {
-                    try {
-                        postService.addPost(post.getPostId(), post.getThreadId());
-                    } catch (PostParseException e) {
-                        log.error(String.format("Failed to add post %s", post.getPostId()), e);
-                        throw new ServerErrorException(String.format("Failed to add post %s", post.getPostId()));
-                    }
-                });
+                threadPoolService.getGeneralExecutor().submit(new AddPostRunnable(post.getPostId(), post.getThreadId()));
             }
         }
     }
@@ -239,11 +205,12 @@ public class PostRestEndpoint {
     public List<MultiPostItem> grab(@PathVariable("threadId") @NonNull String threadId) {
         synchronized (LOCK) {
             Queued queued = dataService.findQueuedByThreadId(threadId).orElseThrow(() -> new NotFoundException(String.format("Unable to find links for threadId = %s", threadId)));
-            try {
-                return postService.getCache().get(queued);
-            } catch (ExecutionException e) {
-                log.error(String.format("Failed to get links for threadId = %s", threadId), e);
+            List<MultiPostItem> multiPostItems = postService.get(queued);
+            if (multiPostItems == null) {
+                log.error(String.format("Failed to get links for threadId = %s", threadId));
                 throw new ServerErrorException(String.format("Failed to get links for threadId = %s", threadId));
+            } else {
+                return multiPostItems;
             }
         }
     }
@@ -256,4 +223,12 @@ public class PostRestEndpoint {
             return threadId;
         }
     }
+
+    @GetMapping("/grab/clear")
+    @ResponseStatus(value = HttpStatus.OK)
+    public void grabClear() {
+        synchronized (LOCK) {
+            dataService.clearQueueLinks();
+        }
+    }
 }

vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java

@@ -8,6 +8,8 @@ import tn.mnlr.vripper.exception.ValidationException;
 import tn.mnlr.vripper.services.SettingsService;
 import tn.mnlr.vripper.web.restendpoints.exceptions.BadRequestException;
 
+import java.util.List;
+
 @RestController
 @Slf4j
 @CrossOrigin(value = "*")
@@ -35,7 +37,7 @@ public class SettingsRestEndpoint {
 
     @PostMapping("/settings")
     @ResponseStatus(value = HttpStatus.OK)
-    public SettingsService.Settings postSettings(@RequestBody SettingsService.Settings settings) throws Exception {
+    public SettingsService.Settings postSettings(@RequestBody SettingsService.Settings settings) {
 
         try {
             this.settingsService.check(settings);
@@ -54,4 +56,10 @@ public class SettingsRestEndpoint {
 
         return settingsService.getSettings();
     }
+
+    @GetMapping("/settings/proxies")
+    @ResponseStatus(value = HttpStatus.OK)
+    public List<String> mirrors() {
+        return settingsService.getProxies();
+    }
 }

vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/DataBroadcast.java

@@ -38,14 +38,16 @@ public class DataBroadcast {
     private final MetadataUpdateEventListener metadataUpdateEventListener;
     private final ImageUpdateEventListener imageUpdateEventListener;
     private final QueuedUpdateEventListener queuedUpdateEventListener;
+    private final EventUpdateEventListener eventUpdateEventListener;
 
     private final QueuedRemoveEventListener queuedRemoveEventListener;
     private final PostRemoveEventListener postRemoveEventListener;
+    private final EventRemoveEventListener eventRemoveEventListener;
 
     private final List<Disposable> disposables = new ArrayList<>();
 
     @Autowired
-    public DataBroadcast(SimpMessagingTemplate template, VGAuthService VGAuthService, GlobalStateService globalStateService, DownloadSpeedService downloadSpeedService, DataService dataService, PostUpdateEventListener postUpdateEventListener, MetadataUpdateEventListener metadataUpdateEventListener, ImageUpdateEventListener imageUpdateEventListener, QueuedUpdateEventListener queuedUpdateEventListener, QueuedRemoveEventListener queuedRemoveEventListener, PostRemoveEventListener postRemoveEventListener) {
+    public DataBroadcast(SimpMessagingTemplate template, VGAuthService VGAuthService, GlobalStateService globalStateService, DownloadSpeedService downloadSpeedService, DataService dataService, PostUpdateEventListener postUpdateEventListener, MetadataUpdateEventListener metadataUpdateEventListener, ImageUpdateEventListener imageUpdateEventListener, QueuedUpdateEventListener queuedUpdateEventListener, EventUpdateEventListener eventUpdateEventListener, QueuedRemoveEventListener queuedRemoveEventListener, PostRemoveEventListener postRemoveEventListener, EventRemoveEventListener eventRemoveEventListener) {
         this.template = template;
         this.VGAuthService = VGAuthService;
         this.globalStateService = globalStateService;
@@ -55,8 +57,10 @@ public class DataBroadcast {
         this.metadataUpdateEventListener = metadataUpdateEventListener;
         this.imageUpdateEventListener = imageUpdateEventListener;
         this.queuedUpdateEventListener = queuedUpdateEventListener;
+        this.eventUpdateEventListener = eventUpdateEventListener;
         this.queuedRemoveEventListener = queuedRemoveEventListener;
         this.postRemoveEventListener = postRemoveEventListener;
+        this.eventRemoveEventListener = eventRemoveEventListener;
     }
 
     @PostConstruct
@@ -86,6 +90,13 @@ public class DataBroadcast {
                 .filter(e -> !e.isEmpty())
                 .subscribe(ids -> template.convertAndSend("/topic/posts", ids.stream().map(dataService::findById).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList())), e -> log.error("Failed to send data to client", e)));
 
+        disposables.add(eventUpdateEventListener.getDataFlux()
+                .map(EventUpdateEvent::getId)
+                .buffer(Duration.of(500, ChronoUnit.MILLIS))
+                .map(HashSet::new)
+                .filter(e -> !e.isEmpty())
+                .subscribe(ids -> template.convertAndSend("/topic/events", ids.stream().map(dataService::findEventById).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList())), e -> log.error("Failed to send data to client", e)));
+
         disposables.add(imageUpdateEventListener.getDataFlux()
                 .map(ImageUpdateEvent::getId)
                 .buffer(Duration.of(500, ChronoUnit.MILLIS))
@@ -124,6 +135,14 @@ public class DataBroadcast {
                 .filter(e -> !e.isEmpty())
                 .subscribe(postIds -> template.convertAndSend("/topic/posts/deleted", postIds), e -> log.error("Failed to send data to client", e))
         );
+
+        disposables.add(eventRemoveEventListener.getDataFlux()
+                .map(EventRemoveEvent::getId)
+                .buffer(Duration.of(500, ChronoUnit.MILLIS))
+                .map(HashSet::new)
+                .filter(e -> !e.isEmpty())
+                .subscribe(postIds -> template.convertAndSend("/topic/events/deleted", postIds), e -> log.error("Failed to send data to client", e))
+        );
     }
 
     @PreDestroy

vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/DataController.java

@@ -5,6 +5,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.messaging.handler.annotation.DestinationVariable;
 import org.springframework.messaging.simp.annotation.SubscribeMapping;
 import org.springframework.stereotype.Controller;
+import tn.mnlr.vripper.jpa.domain.Event;
 import tn.mnlr.vripper.jpa.domain.Image;
 import tn.mnlr.vripper.jpa.domain.Post;
 import tn.mnlr.vripper.jpa.domain.Queued;
@@ -17,8 +18,6 @@ import tn.mnlr.vripper.services.domain.GlobalState;
 
 import java.util.Collection;
 import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
 
 @Controller
 public class DataController {
@@ -53,7 +52,7 @@ public class DataController {
 
     @SubscribeMapping("/posts")
     public Collection<Post> posts() {
-        return StreamSupport.stream(dataService.findAllPosts().spliterator(), false).collect(Collectors.toList());
+        return dataService.findAllPosts();
     }
 
     @SubscribeMapping("/images/{postId}")
@@ -63,7 +62,12 @@ public class DataController {
 
     @SubscribeMapping("/queued")
     public Collection<Queued> queued() {
-        return StreamSupport.stream(dataService.findAllQueued().spliterator(), false).collect(Collectors.toList());
+        return dataService.findAllQueued();
+    }
+
+    @SubscribeMapping("/events")
+    public Collection<Event> events() {
+        return dataService.findAllEvents();
     }
 
     @Getter

vripper-server/src/main/resources/application.properties

@@ -8,7 +8,7 @@ management.endpoints.web.exposure.include=shutdown
 management.endpoint.shutdown.enabled=true
 server.error.include-message=always
 spring.liquibase.change-log=classpath:db/changelog/db.changelog-master.xml
-spring.datasource.url=jdbc:hsqldb:file:${base.dir}/${base.dir.name}/db/xparty;hsqldb.lock_file=false
+spring.datasource.url=jdbc:hsqldb:file:${base.dir}/${base.dir.name}/db/vripper;hsqldb.lock_file=false
 spring.datasource.username=SA
 spring.datasource.password=lEtmEIn
 spring.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver

vripper-server/src/main/resources/db/changelog/db.changelog-master.xml

@@ -2,10 +2,60 @@
 <databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.4.xsd">
-    <changeSet author="sysgen" id="1595764509827-1">
-        <createSequence sequenceName="HIBERNATE_SEQUENCE"/>
-    </changeSet>
-    <changeSet author="sysgen" id="1595764509827-2">
+    <changeSet author="death-claw" id="1613151442-1">
+        <createTable tableName="POST">
+            <column name="ID" type="BIGINT">
+                <constraints nullable="false"/>
+            </column>
+            <column name="DONE" type="INT">
+                <constraints nullable="false"/>
+            </column>
+            <column name="FORUM" type="VARCHAR(500)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="HOSTS" type="VARCHAR(500)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="POST_FOLDER_NAME" type="VARCHAR(500)"/>
+            <column name="POST_ID" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="PREVIEWS" type="VARCHAR(16777216)"/>
+            <column name="SECURITY_TOKEN" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="STATUS" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="THANKED" type="BOOLEAN">
+                <constraints nullable="false"/>
+            </column>
+            <column name="THREAD_ID" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="THREAD_TITLE" type="VARCHAR(500)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="TITLE" type="VARCHAR(500)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="TOTAL" type="INT">
+                <constraints nullable="false"/>
+            </column>
+            <column name="URL" type="VARCHAR(3000)">
+                <constraints nullable="false"/>
+            </column>
+        </createTable>
+        <createTable tableName="METADATA">
+            <column name="POST_ID_REF" type="BIGINT">
+                <constraints nullable="false"/>
+            </column>
+            <column name="POST_ID" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="POSTED_BY" type="VARCHAR(255)"/>
+            <column name="RESOLVED_NAMES" type="VARCHAR(16777216)"/>
+        </createTable>
         <createTable tableName="IMAGE">
             <column name="ID" type="BIGINT">
                 <constraints nullable="false"/>
@@ -31,61 +81,10 @@
             <column name="URL" type="VARCHAR(3000)">
                 <constraints nullable="false"/>
             </column>
-            <column name="POST_ID_REF" type="BIGINT"/>
-        </createTable>
-    </changeSet>
-    <changeSet author="sysgen" id="1595764509827-3">
-        <createTable tableName="METADATA">
-            <column name="ID" type="BIGINT">
-                <constraints nullable="false"/>
-            </column>
-            <column name="POSTED_BY" type="VARCHAR(255)"/>
-            <column name="RESOLVED_NAMES" type="VARCHAR(16777216)"/>
             <column name="POST_ID_REF" type="BIGINT">
                 <constraints nullable="false"/>
             </column>
         </createTable>
-    </changeSet>
-    <changeSet author="sysgen" id="1595764509827-4">
-        <createTable tableName="POST">
-            <column name="ID" type="BIGINT">
-                <constraints nullable="false"/>
-            </column>
-            <column name="DONE" type="INT">
-                <constraints nullable="false"/>
-            </column>
-            <column name="FORUM" type="VARCHAR(500)">
-                <constraints nullable="false"/>
-            </column>
-            <column name="HOSTS" type="VARCHAR(500)">
-                <constraints nullable="false"/>
-            </column>
-            <column name="POST_FOLDER_NAME" type="VARCHAR(500)"/>
-            <column name="POST_ID" type="VARCHAR(255)"/>
-            <column name="PREVIEWS" type="VARCHAR(16777216)"/>
-            <column name="SECURITY_TOKEN" type="VARCHAR(255)">
-                <constraints nullable="false"/>
-            </column>
-            <column name="STATUS" type="VARCHAR(255)">
-                <constraints nullable="false"/>
-            </column>
-            <column name="THANKED" type="BOOLEAN"/>
-            <column name="THREAD_ID" type="VARCHAR(255)">
-                <constraints nullable="false"/>
-            </column>
-            <column name="THREAD_TITLE" type="VARCHAR(500)"/>
-            <column name="TITLE" type="VARCHAR(500)">
-                <constraints nullable="false"/>
-            </column>
-            <column name="TOTAL" type="INT">
-                <constraints nullable="false"/>
-            </column>
-            <column name="URL" type="VARCHAR(3000)">
-                <constraints nullable="false"/>
-            </column>
-        </createTable>
-    </changeSet>
-    <changeSet author="sysgen" id="1595764509827-5">
         <createTable tableName="QUEUED">
             <column name="ID" type="BIGINT">
                 <constraints nullable="false"/>
@@ -104,69 +103,62 @@
                 <constraints nullable="false"/>
             </column>
         </createTable>
+        <createTable tableName="EVENT">
+            <column name="ID" type="BIGINT">
+                <constraints nullable="false"/>
+            </column>
+            <column name="TYPE" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="STATUS" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="TIME" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="MESSAGE" type="VARCHAR(16777216)">
+                <constraints nullable="false"/>
+            </column>
+        </createTable>
     </changeSet>
-    <changeSet author="sysgen" id="1595764509827-6">
-        <addPrimaryKey columnNames="ID" constraintName="SYS_PK_10092" tableName="IMAGE"/>
+    <changeSet author="death-claw" id="1613151442-2">
+        <createSequence sequenceName="SEQ_IMAGE" incrementBy="1" startValue="1"/>
+        <createSequence sequenceName="SEQ_POST" incrementBy="1" startValue="1"/>
+        <createSequence sequenceName="SEQ_QUEUED" incrementBy="1" startValue="1"/>
+        <createSequence sequenceName="SEQ_EVENT" incrementBy="1" startValue="1"/>
     </changeSet>
-    <changeSet author="sysgen" id="1595764509827-7">
-        <addPrimaryKey columnNames="ID" constraintName="SYS_PK_10110" tableName="METADATA"/>
+    <changeSet author="death-claw" id="1613151442-3">
+        <addPrimaryKey columnNames="ID" constraintName="EVENT_PK" tableName="EVENT"/>
+        <addPrimaryKey columnNames="ID" constraintName="IMAGE_PK" tableName="IMAGE"/>
+        <addPrimaryKey columnNames="POST_ID_REF" constraintName="METADATA_PK" tableName="METADATA"/>
+        <addPrimaryKey columnNames="ID" constraintName="POST_PK" tableName="POST"/>
+        <addPrimaryKey columnNames="ID" constraintName="QUEUED_PK" tableName="QUEUED"/>
     </changeSet>
-    <changeSet author="sysgen" id="1595764509827-8">
-        <addPrimaryKey columnNames="ID" constraintName="SYS_PK_10116" tableName="POST"/>
-    </changeSet>
-    <changeSet author="sysgen" id="1595764509827-9">
-        <addPrimaryKey columnNames="ID" constraintName="SYS_PK_10138" tableName="QUEUED"/>
-    </changeSet>
-    <changeSet author="sysgen" id="1595764509827-10">
+    <changeSet author="death-claw" id="1613151442-4">
         <createIndex indexName="IMAGE_POST_ID_IDX" tableName="IMAGE">
             <column name="POST_ID"/>
         </createIndex>
-    </changeSet>
-    <changeSet author="sysgen" id="1595764509827-11">
         <createIndex indexName="IMAGE_STATUS_IDX" tableName="IMAGE">
             <column name="STATUS"/>
         </createIndex>
-    </changeSet>
-    <changeSet author="sysgen" id="1595764509827-12">
         <createIndex indexName="POST_POST_ID_IDX" tableName="POST">
             <column name="POST_ID"/>
         </createIndex>
-    </changeSet>
-    <changeSet author="sysgen" id="1595764509827-13">
         <createIndex indexName="POST_STATUS_IDX" tableName="POST">
             <column name="STATUS"/>
         </createIndex>
-    </changeSet>
-    <changeSet author="sysgen" id="1595764509827-14">
         <createIndex indexName="QUEUED_THREAD_ID_IDX" tableName="QUEUED">
             <column name="THREAD_ID"/>
         </createIndex>
     </changeSet>
-    <changeSet author="sysgen" id="1595764509827-15">
-        <createIndex indexName="SYS_IDX_IMAGE_POST_ID_REF_POST_ID_FK_10150" tableName="IMAGE">
-            <column name="POST_ID_REF"/>
-        </createIndex>
-    </changeSet>
-    <changeSet author="sysgen" id="1595764509827-16">
-        <createIndex indexName="SYS_IDX_METADATA_POST_ID_REF_POST_ID_FK_10160" tableName="METADATA">
-            <column name="POST_ID_REF"/>
-        </createIndex>
-    </changeSet>
-    <changeSet author="sysgen" id="1595764509827-17">
+    <changeSet author="death-claw" id="1613151442-5">
         <addForeignKeyConstraint baseColumnNames="POST_ID_REF" baseTableName="IMAGE"
                                  constraintName="IMAGE_POST_ID_REF_POST_ID_FK" deferrable="false"
                                  initiallyDeferred="false" onDelete="NO ACTION" onUpdate="NO ACTION"
                                  referencedColumnNames="ID" referencedTableName="POST"/>
-    </changeSet>
-    <changeSet author="sysgen" id="1595764509827-18">
         <addForeignKeyConstraint baseColumnNames="POST_ID_REF" baseTableName="METADATA"
                                  constraintName="METADATA_POST_ID_REF_POST_ID_FK" deferrable="false"
                                  initiallyDeferred="false" onDelete="NO ACTION" onUpdate="NO ACTION"
                                  referencedColumnNames="ID" referencedTableName="POST"/>
     </changeSet>
-    <changeSet id="1600798657000-01" author="sysgen">
-        <createIndex tableName="POST" indexName="POST_UQ_POST_ID_IDX">
-            <column name="POST_ID"/>
-        </createIndex>
-    </changeSet>
 </databaseChangeLog>

vripper-server/src/main/resources/proxies.json

@@ -0,0 +1,8 @@
+[
+  "https://planetviper.club",
+  "https://viperbb.rocks",
+  "https://viperkats.eu",
+  "https://viperohilia.art",
+  "https://viperproxy.org",
+  "https://vipervault.link"
+]

vripper-ui/angular.json

@@ -26,10 +26,17 @@
             "assets": [
               "src/favicon.ico",
               "src/assets",
-              { "glob": "mdi.svg", "input": "./node_modules/@mdi/angular-material", "output": "./assets" }
+              {
+                "glob": "mdi.svg",
+                "input": "./node_modules/@mdi/angular-material",
+                "output": "./assets"
+              }
             ],
             "styles": [
               "src/styles.scss"
+            ],
+            "allowedCommonJsDependencies": [
+              "@stomp/rx-stomp"
             ]
           },
           "configurations": {

vripper-ui/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-ui",
-  "version": "3.2.3",
+  "version": "3.3.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

vripper-ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vripper-ui",
-  "version": "3.2.3",
+  "version": "3.3.0",
   "scripts": {
     "ng": "ng",
     "start": "ng serve",

vripper-ui/pom.xml

@@ -5,7 +5,7 @@
   <parent>
     <groupId>tn.mnlr</groupId>
     <artifactId>vripper</artifactId>
-    <version>3.2.3</version>
+    <version>3.3.0</version>
   </parent>
   <artifactId>vripper-ui</artifactId>
   <name>vripper-ui</name>

vripper-ui/src/app/app.module.ts

@@ -27,6 +27,8 @@ import {AlternativeTitleComponent} from './posts/alternative-title/alternative-t
 import {NgxElectronModule} from 'ngx-electron';
 import {MatIconRegistry} from '@angular/material/icon';
 import {DomSanitizer} from '@angular/platform-browser';
+import {EventLogComponent} from './event-log/event-log.component';
+import {EventLogMessageDialogComponent} from './event-log/message-dialog/event-log-message-dialog.component';
 
 @NgModule({
   declarations: [
@@ -45,6 +47,8 @@ import {DomSanitizer} from '@angular/platform-browser';
     MultiPostGridComponent,
     PostContextMenuComponent,
     AlternativeTitleComponent,
+    EventLogComponent,
+    EventLogMessageDialogComponent
   ],
   imports: [
     BrowserAnimationsModule,

vripper-ui/src/app/domain/event.model.ts

@@ -0,0 +1,10 @@
+export class EventLog {
+  constructor(
+    public id: number,
+    public type: string,
+    public status: string,
+    public time: string,
+    public message: string
+  ) {
+  }
+}

vripper-ui/src/app/domain/settings.model.ts

@@ -12,4 +12,6 @@ export interface Settings {
   resolveTitle: boolean;
   connectionTimeout: number;
   maxAttempts: number;
+  vProxy: string;
+  maxEventLog: number;
 }

vripper-ui/src/app/event-log/event-log.component.html

@@ -0,0 +1,4 @@
+<div style="width: 100%; height: 100%">
+  <ag-grid-angular [gridOptions]="gridOptions" class="ag-theme-alpine" style="width: 100%; height: 100%">
+  </ag-grid-angular>
+</div>

vripper-ui/src/app/event-log/event-log.component.ts

@@ -0,0 +1,106 @@
+import {ChangeDetectionStrategy, Component, NgZone, OnDestroy} from '@angular/core';
+import {GridOptions} from 'ag-grid-community';
+import {WsConnectionService} from '../services/ws-connection.service';
+import {EventLogDatasource} from './event-log.datasource';
+import {StatusRendererNative} from '../grid-custom-cells/status-renderer.native';
+import {EventLogService} from '../services/event-log.service';
+import {EventMessageRendererNative} from '../grid-custom-cells/event-message-renderer.native';
+
+@Component({
+  selector: 'app-event-log',
+  templateUrl: './event-log.component.html',
+  changeDetection: ChangeDetectionStrategy.OnPush
+})
+export class EventLogComponent implements OnDestroy {
+  gridOptions: GridOptions;
+  dataSource: EventLogDatasource;
+
+  constructor(
+    private wsConnection: WsConnectionService,
+    private zone: NgZone,
+    private eventLogService: EventLogService
+  ) {
+    this.gridOptions = <GridOptions>{
+      columnDefs: [
+        {
+          headerName: 'Time',
+          field: 'time',
+          sort: 'desc',
+          width: 250,
+          maxWidth: 250
+        }, {
+          headerName: 'Type',
+          field: 'type',
+          valueGetter: (params => {
+            switch (params.data.type) {
+              case 'POST':
+                return 'New gallery';
+              case 'QUEUED':
+                return 'New multi-post link';
+              case 'THANKS':
+                return 'Leave thanks';
+              case 'SCAN':
+                return 'Links scan';
+              case 'METADATA':
+                return 'Fetching metadata';
+              case 'METADATA_CACHE_MISS':
+                return 'Loading post metadata';
+              case 'QUEUED_CACHE_MISS':
+                return 'Loading multi-post links';
+              case 'DOWNLOAD':
+                return 'Download';
+              default:
+                return params.data.type;
+            }
+          }),
+          width: 250,
+          maxWidth: 250
+        }, {
+          headerName: 'Status',
+          field: 'status',
+          cellRenderer: 'nativeStatusCellRenderer',
+          width: 150,
+          maxWidth: 150
+        }, {
+          headerName: 'Message',
+          field: 'message',
+          flex: 1,
+          cellRenderer: 'messageCellRenderer',
+          cellRendererParams: {
+            eventLogService: this.eventLogService
+          },
+        }
+      ],
+      defaultColDef: {
+        sortable: true,
+        resizable: true
+      },
+      rowHeight: 26,
+      headerHeight: 35,
+      animateRows: true,
+      rowSelection: 'single',
+      rowDeselection: true,
+      rowData: [],
+      components: {
+        nativeStatusCellRenderer: StatusRendererNative,
+        messageCellRenderer: EventMessageRendererNative,
+      },
+      overlayLoadingTemplate: '<span></span>',
+      overlayNoRowsTemplate: '<span></span>',
+      getRowNodeId: data => data['id'],
+      onGridReady: () => {
+        this.eventLogService.setGridApi(this.gridOptions.api);
+        this.dataSource = new EventLogDatasource(this.wsConnection, this.gridOptions, this.zone);
+        this.dataSource.connect();
+      },
+      onRowDataUpdated: () => this.eventLogService.setCount(this.gridOptions.api.getDisplayedRowCount()),
+      onRowDataChanged: () => this.eventLogService.setCount(this.gridOptions.api.getDisplayedRowCount()),
+    };
+  }
+
+  ngOnDestroy(): void {
+    if (this.dataSource) {
+      this.dataSource.disconnect();
+    }
+  }
+}

vripper-ui/src/app/event-log/event-log.datasource.ts

@@ -0,0 +1,48 @@
+import {Subscription} from 'rxjs';
+import {WsConnectionService} from '../services/ws-connection.service';
+import {GridOptions, RowNode} from 'ag-grid-community';
+import {NgZone} from '@angular/core';
+import {EventLog} from '../domain/event.model';
+
+export class EventLogDatasource {
+  subscriptions: Subscription[] = [];
+
+  constructor(private ws: WsConnectionService, private gridOptions: GridOptions, private zone: NgZone) {
+  }
+
+  connect() {
+    this.subscriptions.push(this.ws.events$.subscribe((e: EventLog[]) => {
+      this.zone.run(() => {
+        const toAdd = [];
+        const toUpdate = [];
+        e.forEach(v => {
+          if (this.gridOptions.api.getRowNode(v.id.toString(10)) == null) {
+            toAdd.push(v);
+          } else {
+            toUpdate.push(v);
+          }
+        });
+        this.gridOptions.api.applyTransaction({update: toUpdate, add: toAdd});
+      });
+    }));
+
+    this.subscriptions.push(this.ws.eventsRemove$.subscribe((e: number[]) => {
+      this.zone.run(() => {
+        const toRemove = [];
+        e.forEach(v => {
+          const rowNode: RowNode = this.gridOptions.api.getRowNode(v.toString(10));
+          if (rowNode != null) {
+            toRemove.push(rowNode.data);
+          }
+          return;
+        });
+        this.gridOptions.api.applyTransaction({remove: toRemove});
+      });
+    }));
+  }
+
+  disconnect() {
+    console.log('Disconnecting from events datasource');
+    this.subscriptions.forEach(e => e.unsubscribe());
+  }
+}

vripper-ui/src/app/event-log/message-dialog/event-log-message-dialog.component.html

@@ -0,0 +1,10 @@
+<div class="dialog-container" fxLayout="column" style="height: 100%;">
+  <h2 class="no-wrap" mat-dialog-title>Message</h2>
+  <mat-dialog-content fxFlex="grow">
+    <pre>{{data}}</pre>
+  </mat-dialog-content>
+  <mat-dialog-actions align="end" fxFlex="nogrow">
+    <button (click)="close()" color="primary" mat-raised-button>Close</button>
+  </mat-dialog-actions>
+</div>
+

vripper-ui/src/app/event-log/message-dialog/event-log-message-dialog.component.ts

@@ -0,0 +1,19 @@
+import {ChangeDetectionStrategy, Component, Inject, NgZone} from '@angular/core';
+import {MAT_DIALOG_DATA, MatDialogRef} from '@angular/material/dialog';
+
+@Component({
+  selector: 'app-event-log-message-dialog',
+  templateUrl: './event-log-message-dialog.component.html',
+  changeDetection: ChangeDetectionStrategy.OnPush
+})
+export class EventLogMessageDialogComponent {
+
+  constructor(public dialogRef: MatDialogRef<EventLogMessageDialogComponent>,
+              @Inject(MAT_DIALOG_DATA) public data: string,
+              private ngZone: NgZone) {
+  }
+
+  close() {
+    this.ngZone.run(() => this.dialogRef.close());
+  }
+}

vripper-ui/src/app/grid-custom-cells/collector-actions-renderer.native.ts

@@ -32,14 +32,14 @@ export class CollectorActionsRendererNative implements ICellRendererComp {
     this.selectButton = document.createElement('button');
     this.selectButton.classList.add('mat-icon-button', 'mat-button-base', 'cell-icon');
     this.selectButton.setAttribute('style', 'width: auto; height: 24px; display: flex; align-items: center');
-    this.selectButton.innerHTML = `<svg style="width:24px;height:24px" viewBox="0 0 24 24"><path fill="currentColor" d="M20,2H8A2,2 0 0,0 6,4V16A2,2 0 0,0 8,18H20A2,2 0 0,0 22,16V4A2,2 0 0,0 20,2M20,16H8V4H20V16M16,20V22H4A2,2 0 0,1 2,20V7H4V20H16M18.53,8.06L17.47,7L12.59,11.88L10.47,9.76L9.41,10.82L12.59,14L18.53,8.06Z" /></svg><span>Select</span>`;
+    this.selectButton.innerHTML = `<svg style="width:24px;height:24px" viewBox="0 0 24 24"><path fill="currentColor" d="M20,2H8A2,2 0 0,0 6,4V16A2,2 0 0,0 8,18H20A2,2 0 0,0 22,16V4A2,2 0 0,0 20,2M20,16H8V4H20V16M16,20V22H4A2,2 0 0,1 2,20V7H4V20H16M18.53,8.06L17.47,7L12.59,11.88L10.47,9.76L9.41,10.82L12.59,14L18.53,8.06Z" /></svg><span style="margin-left: 5px">Select</span>`;
     this.selectSpan.append(this.selectButton);
 
     this.deleteSpan = document.createElement('span');
     this.deleteButton = document.createElement('button');
     this.deleteButton.classList.add('mat-icon-button', 'mat-button-base', 'cell-icon');
     this.deleteButton.setAttribute('style', 'width: auto; height: 24px; display: flex; align-items: center');
-    this.deleteButton.innerHTML = `<svg style="width:24px;height:24px" viewBox="0 0 24 24"><path fill="currentColor" d="M19,4H15.5L14.5,3H9.5L8.5,4H5V6H19M6,19A2,2 0 0,0 8,21H16A2,2 0 0,0 18,19V7H6V19Z" /></svg><span>Remove</span>`;
+    this.deleteButton.innerHTML = `<svg style="width:24px;height:24px" viewBox="0 0 24 24"><path fill="currentColor" d="M19,4H15.5L14.5,3H9.5L8.5,4H5V6H19M6,19A2,2 0 0,0 8,21H16A2,2 0 0,0 18,19V7H6V19Z" /></svg><span style="margin-left: 5px">Remove</span>`;
     this.deleteSpan.append(this.deleteButton);
 
     this.gui.append(this.selectSpan, this.deleteSpan);

vripper-ui/src/app/grid-custom-cells/event-message-renderer.native.ts

@@ -0,0 +1,50 @@
+import {GridApi, ICellRendererComp, ICellRendererParams} from 'ag-grid-community';
+import {EventLog} from '../domain/event.model';
+import {EventLogService} from '../services/event-log.service';
+
+export class EventMessageRendererNative implements ICellRendererComp {
+  private gui: HTMLElement;
+  private viewSpan: HTMLSpanElement;
+  private viewButton: HTMLButtonElement;
+  private eventLogService: EventLogService;
+  private eventLog: EventLog;
+  private gridApi: GridApi;
+  private text: HTMLSpanElement;
+
+  destroy(): void {
+    this.viewButton.removeEventListener('click', () => this.eventLogService.openDialog(this.eventLog.message));
+  }
+
+  getGui(): HTMLElement {
+    return this.gui;
+  }
+
+  init(params: ICellRendererParams): void {
+    // @ts-ignore
+    this.eventLogService = params.eventLogService;
+    this.eventLog = params.node.data;
+    this.gridApi = params.api;
+    this.gui = document.createElement('div');
+    this.gui.setAttribute('style', 'display: flex; justify-content: space-between;');
+    this.text = document.createElement('span');
+    this.text.classList.add('no-wrap');
+    this.text.textContent = this.eventLog.message;
+    this.viewSpan = document.createElement('span');
+    this.viewButton = document.createElement('button');
+    this.viewButton.classList.add('mat-icon-button', 'mat-button-base', 'cell-icon');
+    this.viewButton.setAttribute('style', 'width: auto; height: 24px; display: flex; align-items: center');
+    this.viewButton.innerHTML = `<svg style="width:24px;height:24px" viewBox="0 0 24 24">
+    <path fill="currentColor" d="M12,9A3,3 0 0,0 9,12A3,3 0 0,0 12,15A3,3 0 0,0 15,12A3,3 0 0,0 12,9M12,17A5,5 0 0,1 7,12A5,5 0 0,1 12,7A5,5 0 0,1 17,12A5,5 0 0,1 12,17M12,4.5C7,4.5 2.73,7.61 1,12C2.73,16.39 7,19.5 12,19.5C17,19.5 21.27,16.39 23,12C21.27,7.61 17,4.5 12,4.5Z" /></svg><span style="margin-left: 5px">View</span>`;
+    this.viewSpan.append(this.viewButton);
+
+    this.gui.append(this.text, this.viewSpan);
+
+    this.viewButton.addEventListener('click', () => this.eventLogService.openDialog(this.eventLog.message));
+  }
+
+  refresh(params: ICellRendererParams): boolean {
+    this.eventLog = params.node.data;
+    this.text.textContent = this.eventLog.message;
+    return true;
+  }
+}

vripper-ui/src/app/grid-custom-cells/status-renderer.native.ts

@@ -61,8 +61,13 @@ export class StatusRendererNative implements ICellRendererComp {
       case 'DOWNLOADING':
         return `<svg style="width:24px;height:24px" viewBox="0 0 24 24">
     <path fill="currentColor" d="M5,20H19V18H5M19,9H15V3H9V9H5L12,16L19,9Z" />
+</svg>`;
+      case 'PROCESSING':
+        return `<svg style="width:24px;height:24px" viewBox="0 0 24 24">
+    <path fill="currentColor" d="M15.9,18.45C17.25,18.45 18.35,17.35 18.35,16C18.35,14.65 17.25,13.55 15.9,13.55C14.54,13.55 13.45,14.65 13.45,16C13.45,17.35 14.54,18.45 15.9,18.45M21.1,16.68L22.58,17.84C22.71,17.95 22.75,18.13 22.66,18.29L21.26,20.71C21.17,20.86 21,20.92 20.83,20.86L19.09,20.16C18.73,20.44 18.33,20.67 17.91,20.85L17.64,22.7C17.62,22.87 17.47,23 17.3,23H14.5C14.32,23 14.18,22.87 14.15,22.7L13.89,20.85C13.46,20.67 13.07,20.44 12.71,20.16L10.96,20.86C10.81,20.92 10.62,20.86 10.54,20.71L9.14,18.29C9.05,18.13 9.09,17.95 9.22,17.84L10.7,16.68L10.65,16L10.7,15.31L9.22,14.16C9.09,14.05 9.05,13.86 9.14,13.71L10.54,11.29C10.62,11.13 10.81,11.07 10.96,11.13L12.71,11.84C13.07,11.56 13.46,11.32 13.89,11.15L14.15,9.29C14.18,9.13 14.32,9 14.5,9H17.3C17.47,9 17.62,9.13 17.64,9.29L17.91,11.15C18.33,11.32 18.73,11.56 19.09,11.84L20.83,11.13C21,11.07 21.17,11.13 21.26,11.29L22.66,13.71C22.75,13.86 22.71,14.05 22.58,14.16L21.1,15.31L21.15,16L21.1,16.68M6.69,8.07C7.56,8.07 8.26,7.37 8.26,6.5C8.26,5.63 7.56,4.92 6.69,4.92A1.58,1.58 0 0,0 5.11,6.5C5.11,7.37 5.82,8.07 6.69,8.07M10.03,6.94L11,7.68C11.07,7.75 11.09,7.87 11.03,7.97L10.13,9.53C10.08,9.63 9.96,9.67 9.86,9.63L8.74,9.18L8,9.62L7.81,10.81C7.79,10.92 7.7,11 7.59,11H5.79C5.67,11 5.58,10.92 5.56,10.81L5.4,9.62L4.64,9.18L3.5,9.63C3.41,9.67 3.3,9.63 3.24,9.53L2.34,7.97C2.28,7.87 2.31,7.75 2.39,7.68L3.34,6.94L3.31,6.5L3.34,6.06L2.39,5.32C2.31,5.25 2.28,5.13 2.34,5.03L3.24,3.47C3.3,3.37 3.41,3.33 3.5,3.37L4.63,3.82L5.4,3.38L5.56,2.19C5.58,2.08 5.67,2 5.79,2H7.59C7.7,2 7.79,2.08 7.81,2.19L8,3.38L8.74,3.82L9.86,3.37C9.96,3.33 10.08,3.37 10.13,3.47L11.03,5.03C11.09,5.13 11.07,5.25 11,5.32L10.03,6.06L10.06,6.5L10.03,6.94Z" />
 </svg>`;
       case 'COMPLETE':
+      case 'DONE':
         return `<svg style="width:24px;height:24px" viewBox="0 0 24 24">
     <path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z" />
 </svg>`;
@@ -86,8 +91,10 @@ export class StatusRendererNative implements ICellRendererComp {
       case 'PENDING':
         return 'pending';
       case 'DOWNLOADING':
+      case 'PROCESSING':
         return 'downloading';
       case 'COMPLETE':
+      case 'DONE':
         return 'complete';
       case 'ERROR':
         return 'error';

vripper-ui/src/app/home/home.component.html

@@ -1,15 +1,31 @@
-<mat-tab-group animationDuration="0" dynamicHeight style="height: 100%; margin-bottom: 20px;">
-  <mat-tab label="Downloads">
+<mat-tab-group (selectedTabChange)="onTabChange($event)" animationDuration="0" dynamicHeight
+               style="height: 100%; margin-bottom: 20px;">
+  <mat-tab>
+    <ng-template mat-tab-label>
+      <span>Downloads</span>
+      <ng-container *ngIf="postsService.count | async as count">
+        <mat-icon [matBadgeHidden]="count < 1" [matBadge]="count" class="icon">photo_album</mat-icon>
+      </ng-container>
+    </ng-template>
     <app-posts></app-posts>
   </mat-tab>
   <mat-tab>
     <ng-template mat-tab-label>
       <span>Multi-post links</span>
       <ng-container *ngIf="linkCollectorService.count | async as count">
-        <mat-icon [matBadgeHidden]="count < 1" [matBadge]="count">link</mat-icon>
+        <mat-icon [matBadgeHidden]="count < 1" [matBadge]="count" class="icon">link</mat-icon>
       </ng-container>
     </ng-template>
     <app-grab-queue></app-grab-queue>
   </mat-tab>
+  <mat-tab>
+    <ng-template mat-tab-label>
+      <span>Event Log</span>
+      <ng-container *ngIf="eventLogService.count | async as count">
+        <mat-icon [matBadgeHidden]="count < 1" [matBadge]="count" class="icon">grid_on</mat-icon>
+      </ng-container>
+    </ng-template>
+    <app-event-log></app-event-log>
+  </mat-tab>
 </mat-tab-group>
 

vripper-ui/src/app/home/home.component.scss

@@ -47,3 +47,7 @@ table {
     align-self: start;
   }
 }
+
+.icon {
+  margin-left: 5px;
+}

vripper-ui/src/app/home/home.component.ts

@@ -6,6 +6,10 @@ import {ChangeDetectionStrategy, Component, NgZone, OnDestroy, OnInit} from '@an
 import {MatDialog} from '@angular/material/dialog';
 import {MatSnackBar} from '@angular/material/snack-bar';
 import {HttpClient} from '@angular/common/http';
+import {EventLogService} from '../services/event-log.service';
+import {PostsService} from '../services/posts.service';
+import {MatTabChangeEvent} from '@angular/material/tabs';
+import {HomeTabsService} from '../services/home-tabs.service';
 
 @Component({
   selector: 'app-home',
@@ -23,7 +27,10 @@ export class HomeComponent implements OnInit, OnDestroy {
     private serverService: ServerService,
     private _snackBar: MatSnackBar,
     private ngZone: NgZone,
-    public linkCollectorService: LinkCollectorService
+    public linkCollectorService: LinkCollectorService,
+    public eventLogService: EventLogService,
+    public postsService: PostsService,
+    public homeTabsService: HomeTabsService
   ) {
   }
 
@@ -53,4 +60,8 @@ export class HomeComponent implements OnInit, OnDestroy {
       this.clipboardSub.unsubscribe();
     }
   }
+
+  onTabChange($event: MatTabChangeEvent) {
+    this.homeTabsService.setIndex($event.index);
+  }
 }

vripper-ui/src/app/multi-post-grid/multi-post-grid-data.source.ts

@@ -47,6 +47,7 @@ export class MultiPostGridDataSource {
   }
 
   disconnect() {
+    console.log('Disconnecting from link collector datasource');
     this.subscriptions.forEach(e => e.unsubscribe());
   }
 }

vripper-ui/src/app/multi-post-grid/multi-post-grid.component.ts

@@ -67,14 +67,18 @@ export class MultiPostGridComponent implements OnDestroy {
       overlayNoRowsTemplate: '<span></span>',
       getRowNodeId: data => data['threadId'],
       onGridReady: () => {
+        this.linkCollectorService.setGridApi(this.gridOptions.api);
         this.dataSource = new MultiPostGridDataSource(this.wsConnection, this.gridOptions, this.zone);
         this.dataSource.connect();
       },
       onRowDataUpdated: () => this.linkCollectorService.setCount(this.gridOptions.api.getDisplayedRowCount()),
+      onRowDataChanged: () => this.linkCollectorService.setCount(this.gridOptions.api.getDisplayedRowCount()),
     };
   }
 
   ngOnDestroy(): void {
-    this.dataSource.disconnect();
+    if (this.dataSource) {
+      this.dataSource.disconnect();
+    }
   }
 }

vripper-ui/src/app/posts/posts.component.ts

@@ -3,7 +3,6 @@ import {ChangeDetectionStrategy, Component, NgZone, OnDestroy} from '@angular/co
 import {PostsDataSource} from './posts.datasource';
 import {WsConnectionService} from '../services/ws-connection.service';
 import {GridOptions} from 'ag-grid-community';
-import {Subject} from 'rxjs';
 import {PostContextMenuService} from '../services/post-context-menu.service';
 import {PostProgressRendererNative} from '../grid-custom-cells/post-progress-renderer.native';
 import {PostStatusRendererNative} from '../grid-custom-cells/post-status-renderer.native';
@@ -19,7 +18,6 @@ import {PostsService} from '../services/posts.service';
   changeDetection: ChangeDetectionStrategy.OnPush
 })
 export class PostsComponent implements OnDestroy {
-  dialogOpen: Subject<boolean> = new Subject();
   gridOptions: GridOptions;
   dataSource: PostsDataSource;
 
@@ -30,7 +28,8 @@ export class PostsComponent implements OnDestroy {
     private postsDataService: PostsService,
     private contextMenuService: PostContextMenuService,
     private overlayPositionBuilder: OverlayPositionBuilder,
-    private overlay: Overlay
+    private overlay: Overlay,
+    private postsService: PostsService
   ) {
     this.gridOptions = <GridOptions>{
       columnDefs: [
@@ -110,11 +109,15 @@ export class PostsComponent implements OnDestroy {
         this.dataSource.connect();
       },
       onSelectionChanged: () => this.selectionService.onSelectionChanged(this.gridOptions.api.getSelectedNodes()),
-      onBodyScroll: () => this.contextMenuService.closePostContextMenu()
+      onBodyScroll: () => this.contextMenuService.closePostContextMenu(),
+      onRowDataUpdated: () => this.postsService.setCount(this.gridOptions.api.getDisplayedRowCount()),
+      onRowDataChanged: () => this.postsService.setCount(this.gridOptions.api.getDisplayedRowCount()),
     };
   }
 
   ngOnDestroy(): void {
-    this.dataSource.disconnect();
+    if (this.dataSource) {
+      this.dataSource.disconnect();
+    }
   }
 }

vripper-ui/src/app/services/event-log.service.ts

@@ -0,0 +1,53 @@
+import {Injectable} from '@angular/core';
+import {BehaviorSubject, Observable, Subject} from 'rxjs';
+import {GridApi} from 'ag-grid-community';
+import {MatDialog, MatDialogConfig} from '@angular/material/dialog';
+import {EventLogMessageDialogComponent} from '../event-log/message-dialog/event-log-message-dialog.component';
+
+@Injectable({
+  providedIn: 'root'
+})
+export class EventLogService {
+  private api: GridApi;
+
+  private _count$: Subject<number> = new BehaviorSubject(0);
+
+  constructor(public dialog: MatDialog) {
+  }
+
+  get count(): Observable<number> {
+    return this._count$.asObservable();
+  }
+
+  setCount(count: number) {
+    this._count$.next(count);
+  }
+
+  public setGridApi(api: GridApi) {
+    this.api = api;
+  }
+
+  search(event) {
+    if (this.api) {
+      this.api.setQuickFilter(event);
+    }
+  }
+
+  clear() {
+    if (this.api) {
+      this.api.setRowData([]);
+    }
+  }
+
+  openDialog(message: string) {
+    console.log(message);
+    const config: MatDialogConfig<string> = {
+      width: '70%',
+      height: '70%',
+      maxWidth: '100vw',
+      maxHeight: '100vh',
+      data: message
+    };
+    this.dialog.open(EventLogMessageDialogComponent, config);
+  }
+}

vripper-ui/src/app/services/home-tabs.service.ts

@@ -0,0 +1,18 @@
+import {Injectable} from '@angular/core';
+import {BehaviorSubject, Observable, Subject} from 'rxjs';
+
+@Injectable({
+  providedIn: 'root'
+})
+export class HomeTabsService {
+
+  private _index$: Subject<number> = new BehaviorSubject(0);
+
+  get index(): Observable<number> {
+    return this._index$.asObservable();
+  }
+
+  setIndex(count: number) {
+    this._index$.next(count);
+  }
+}

vripper-ui/src/app/services/link-collector.service.ts

@@ -1,10 +1,12 @@
 import {Injectable} from '@angular/core';
 import {BehaviorSubject, Observable, Subject} from 'rxjs';
+import {GridApi} from 'ag-grid-community';
 
 @Injectable({
   providedIn: 'root'
 })
 export class LinkCollectorService {
+  private api: GridApi;
 
   private _count$: Subject<number> = new BehaviorSubject(0);
 
@@ -12,7 +14,23 @@ export class LinkCollectorService {
     return this._count$.asObservable();
   }
 
+  public setGridApi(api: GridApi) {
+    this.api = api;
+  }
+
+  search(event) {
+    if (this.api) {
+      this.api.setQuickFilter(event);
+    }
+  }
+
   setCount(count: number) {
     this._count$.next(count);
   }
+
+  clear() {
+    if (this.api) {
+      this.api.setRowData([]);
+    }
+  }
 }

vripper-ui/src/app/services/posts.service.ts

@@ -1,5 +1,6 @@
 import {Injectable} from '@angular/core';
 import {GridApi} from 'ag-grid-community';
+import {BehaviorSubject, Observable, Subject} from 'rxjs';
 
 @Injectable({
   providedIn: 'root'
@@ -7,6 +8,16 @@ import {GridApi} from 'ag-grid-community';
 export class PostsService {
   private api: GridApi;
 
+  private _count$: Subject<number> = new BehaviorSubject(0);
+
+  get count(): Observable<number> {
+    return this._count$.asObservable();
+  }
+
+  setCount(count: number) {
+    this._count$.next(count);
+  }
+
   public setGridApi(api: GridApi) {
     this.api = api;
   }
@@ -23,6 +34,8 @@ export class PostsService {
   }
 
   search(event) {
-    this.api.setQuickFilter(event);
+    if (this.api) {
+      this.api.setQuickFilter(event);
+    }
   }
 }

vripper-ui/src/app/services/ws-connection.service.ts

@@ -12,6 +12,7 @@ import {GlobalState} from '../domain/global-state.model';
 import {map, share} from 'rxjs/operators';
 import {RxStomp, RxStompConfig, RxStompState} from '@stomp/rx-stomp';
 import {ElectronService} from 'ngx-electron';
+import {EventLog} from '../domain/event.model';
 
 @Injectable({
   providedIn: 'root'
@@ -22,10 +23,12 @@ export class WsConnectionService {
   private speed: Observable<DownloadSpeed>;
   private user: Observable<LoggedUser>;
   private posts: Observable<Array<Post>>;
+  private events: Observable<Array<EventLog>>;
   private postDetails: Observable<Array<Photo>>;
   private multiPostModels: Observable<Array<MultiPostModel>>;
   private queuedRemove: Observable<Array<string>>;
   private postsRemove: Observable<Array<string>>;
+  private eventsRemove: Observable<Array<number>>;
   private postIdDetails: string;
 
   private connectionState$: Subject<RxStompState> = new BehaviorSubject(RxStompState.CLOSED);
@@ -59,6 +62,14 @@ export class WsConnectionService {
     return this.postsRemove;
   }
 
+  get events$(): Observable<EventLog[]> {
+    return this.events;
+  }
+
+  get eventsRemove$(): Observable<number[]> {
+    return this.eventsRemove;
+  }
+
   get queuedRemove$(): Observable<string[]> {
     return this.queuedRemove;
   }
@@ -121,6 +132,13 @@ export class WsConnectionService {
       share()
     );
 
+    this.eventsRemove = this.rxStomp.watch('/topic/events/deleted').pipe(
+      map(e => {
+        return JSON.parse(e.body);
+      }),
+      share()
+    );
+
     this.queuedRemove = this.rxStomp.watch('/topic/queued/deleted').pipe(
       map(e => {
         return JSON.parse(e.body);
@@ -176,6 +194,25 @@ export class WsConnectionService {
       share()
     );
 
+    this.events = this.rxStomp.watch('/topic/events').pipe(
+      map(e => {
+        const events: Array<EventLog> = [];
+        (<Array<any>>JSON.parse(e.body)).forEach(element => {
+          events.push(
+            new EventLog(
+              element.id,
+              element.type,
+              element.status,
+              element.time,
+              element.message
+            )
+          );
+        });
+        return events;
+      }),
+      share()
+    );
+
     this.multiPostModels = this.rxStomp.watch('/topic/queued').pipe(
       map(e => {
         const multiPostModels: Array<MultiPostModel> = [];

vripper-ui/src/app/settings/settings.component.html

@@ -134,6 +134,13 @@
         </mat-tab>
         <mat-tab label="ViperGirls">
           <form [formGroup]="viperGirlsSettingsForm" autocomplete="off" fxLayout="column">
+            <mat-form-field>
+              <mat-label>Select a proxy</mat-label>
+              <mat-select formControlName="vProxy" name="vProxy">
+                <mat-option *ngFor="let mirror of mirrors | async" [value]="mirror">{{mirror}}</mat-option>
+              </mat-select>
+            </mat-form-field>
+
             <mat-slide-toggle color="primary" formControlName="vLogin" name="vLogin">
               Enable ViperGirls Authentication
             </mat-slide-toggle>
@@ -169,6 +176,28 @@
             </section>
           </form>
         </mat-tab>
+        <mat-tab label="Event Log">
+          <form [formGroup]="eventLogSettingsForm" autocomplete="off" fxLayout="column">
+            <section class="full-width" fxLayout="row" fxLayoutAlign="space-between center" fxLayoutGap="5px">
+              <mat-form-field fxFlex="grow">
+                <input
+                  formControlName="maxEventLog"
+                  matInput
+                  max="10000"
+                  min="100"
+                  name="maxEventLog"
+                  placeholder="Maximum records"
+                  required
+                  type="number"
+                />
+              </mat-form-field>
+              <mat-icon
+                matTooltip="Oldest records will be removed once the maximum amount is reached"
+                svgIcon="help-circle-outline">
+              </mat-icon>
+            </section>
+          </form>
+        </mat-tab>
         <mat-tab *ngIf="electronService.isElectronApp" label="Desktop Integration">
           <form [formGroup]="desktopSettingsForm" autocomplete="off">
             <section class="input">
@@ -188,11 +217,12 @@
     <button
       (click)="onSubmit()"
       [disabled]="
-        (desktopSettingsForm.pristine && connectionSettingsForm.pristine && downloadSettingsForm.pristine && viperGirlsSettingsForm.pristine) ||
+        (desktopSettingsForm.pristine && connectionSettingsForm.pristine && downloadSettingsForm.pristine && viperGirlsSettingsForm.pristine && eventLogSettingsForm.pristine) ||
         viperGirlsSettingsForm.invalid ||
         downloadSettingsForm.invalid ||
         connectionSettingsForm.invalid ||
         desktopSettingsForm.invalid ||
+        eventLogSettingsForm.invalid ||
         loading
       "
       color="primary"

vripper-ui/src/app/settings/settings.component.scss

@@ -13,3 +13,7 @@ form {
     margin-right: 5px;
   }
 }
+
+.full-width {
+  width: 100%;
+}

vripper-ui/src/app/settings/settings.component.ts

@@ -8,6 +8,7 @@ import {MatSnackBar} from '@angular/material/snack-bar';
 import {ServerService} from '../services/server-service';
 import {ElectronService} from 'ngx-electron';
 import {Settings} from '../domain/settings.model';
+import {EMPTY, Observable} from 'rxjs';
 import OpenDialogReturnValue = Electron.OpenDialogReturnValue;
 
 @Component({
@@ -20,12 +21,15 @@ export class SettingsComponent implements OnInit {
   loading = false;
   darkTheme = false;
 
+  mirrors: Observable<string[]> = EMPTY;
+
   viperGirlsSettingsForm = new FormGroup({
     vLogin: new FormControl(false),
     vUsername: new FormControl(''),
     vPassword: new FormControl(''),
     vThanks: new FormControl(false),
-    leaveThanksOnStart: new FormControl(false)
+    leaveThanksOnStart: new FormControl(false),
+    vProxy: new FormControl('')
   });
 
   downloadSettingsForm = new FormGroup({
@@ -49,6 +53,10 @@ export class SettingsComponent implements OnInit {
     desktopClipboard: new FormControl(false)
   });
 
+  eventLogSettingsForm = new FormGroup({
+    maxEventLog: new FormControl('')
+  });
+
   constructor(
     private httpClient: HttpClient,
     private _snackBar: MatSnackBar,
@@ -70,12 +78,14 @@ export class SettingsComponent implements OnInit {
 
   ngOnInit() {
     this.darkTheme = this.appService.darkTheme;
+    this.mirrors = this.httpClient.get<string[]>(this.serverService.baseUrl + '/settings/proxies');
     this.httpClient.get<Settings>(this.serverService.baseUrl + '/settings')
       .subscribe(data => {
         this.viperGirlsSettingsForm.reset(data);
         this.downloadSettingsForm.reset(data);
         this.connectionSettingsForm.reset(data);
         this.desktopSettingsForm.reset(data);
+        this.eventLogSettingsForm.reset(data);
       }, error => {
         this._snackBar.open(error?.error?.message || 'Unexpected error, check log file', null, {
           duration: 5000
@@ -107,7 +117,8 @@ export class SettingsComponent implements OnInit {
         ...this.downloadSettingsForm.value,
         ...this.connectionSettingsForm.value,
         darkTheme: this.darkTheme,
-        ...this.desktopSettingsForm.value
+        ...this.desktopSettingsForm.value,
+        ...this.eventLogSettingsForm.value
       })
       .pipe(finalize(() => (this.loading = false)))
       .subscribe(
@@ -119,6 +130,7 @@ export class SettingsComponent implements OnInit {
           this.downloadSettingsForm.reset(data);
           this.connectionSettingsForm.reset(data);
           this.desktopSettingsForm.reset(data);
+          this.eventLogSettingsForm.reset(data);
           this.clipboardService.init(data);
           this.updateSettings(data);
         },

vripper-ui/src/app/toolbar/toolbar.component.html

@@ -4,37 +4,50 @@
       <button (click)="scan()" class="add-button" color="primary" mat-icon-button matTooltip="Add links">
         <mat-icon>add</mat-icon>
       </button>
-      <button (click)="remove()" [disabled]="disableSelection$ | async" mat-icon-button
-              matTooltip="Remove selected">
-        <mat-icon>delete</mat-icon>
-      </button>
-      <button (click)="restart()" [disabled]="disableSelection$ | async" mat-icon-button
-              matTooltip="Start selected">
-        <mat-icon>play_arrow</mat-icon>
-      </button>
-      <button (click)="stop()" [disabled]="disableSelection$ | async" mat-icon-button
-              matTooltip="Stop selected">
-        <mat-icon>pause</mat-icon>
-      </button>
-      <button (click)="rename()" [disabled]="disableSelection$ | async"
-              mat-icon-button
-              matTooltip="Rename selected to first alternative title">
-        <mat-icon>edit</mat-icon>
-      </button>
-      <div class="or-spacer-vertical left">
-        <div class="mask"></div>
-      </div>
-      <button (click)="clear()" mat-icon-button matTooltip="Clear completed">
-        <mat-icon>clear_all</mat-icon>
-      </button>
-      <button (click)="stopAll()" mat-icon-button matTooltip="Stop All">
-        <mat-icon>stop</mat-icon>
-      </button>
+      <ng-container *ngIf="(homeTabsService.index | async) === 0" id="download-actions">
+        <button (click)="remove()" [disabled]="disableSelection$ | async" mat-icon-button
+                matTooltip="Remove selected">
+          <mat-icon>delete</mat-icon>
+        </button>
+        <button (click)="restart()" [disabled]="disableSelection$ | async" mat-icon-button
+                matTooltip="Start selected">
+          <mat-icon>play_arrow</mat-icon>
+        </button>
+        <button (click)="stop()" [disabled]="disableSelection$ | async" mat-icon-button
+                matTooltip="Stop selected">
+          <mat-icon>pause</mat-icon>
+        </button>
+        <button (click)="rename()" [disabled]="disableSelection$ | async"
+                mat-icon-button
+                matTooltip="Rename selected to first alternative title">
+          <mat-icon>edit</mat-icon>
+        </button>
+        <div class="or-spacer-vertical left">
+          <div class="mask"></div>
+        </div>
+        <button (click)="clear()" mat-icon-button matTooltip="Clear completed">
+          <mat-icon>clear_all</mat-icon>
+        </button>
+        <button (click)="stopAll()" mat-icon-button matTooltip="Stop All">
+          <mat-icon>stop</mat-icon>
+        </button>
+      </ng-container>
+      <ng-container *ngIf="(homeTabsService.index | async) === 1" id="link-collector-actions">
+        <button (click)="clearLinkCollector()" mat-icon-button matTooltip="Remove all">
+          <mat-icon>clear_all</mat-icon>
+        </button>
+      </ng-container>
+      <ng-container *ngIf="(homeTabsService.index | async) === 2" id="event-logs-actions">
+        <button (click)="clearEventLogs()" mat-icon-button matTooltip="Clear all">
+          <mat-icon>clear_all</mat-icon>
+        </button>
+      </ng-container>
     </div>
     <div fxFlex="grow" fxHide.lt-md id="global-search">
       <form autocomplete="off">
         <mat-form-field class="search-field">
-          <input (ngModelChange)="search($event)" matInput name="search" ngModel placeholder="Search"/>
+          <input (ngModelChange)="search($event)" [(ngModel)]="searchModel" matInput name="search" ngModel
+                 placeholder="Search"/>
           <mat-icon matSuffix>search</mat-icon>
         </mat-form-field>
       </form>

vripper-ui/src/app/toolbar/toolbar.component.ts

@@ -17,6 +17,9 @@ import {RowNode} from 'ag-grid-community';
 import {RemoveResponse} from '../domain/remove-response.model';
 import {PostId} from '../domain/post-id.model';
 import {PostsService} from '../services/posts.service';
+import {HomeTabsService} from '../services/home-tabs.service';
+import {LinkCollectorService} from '../services/link-collector.service';
+import {EventLogService} from '../services/event-log.service';
 
 @Component({
   selector: 'app-toolbar',
@@ -30,6 +33,8 @@ export class ToolbarComponent implements OnInit, OnDestroy {
   isExtraSmall: Observable<BreakpointState> = this.breakpointObserver.observe(Breakpoints.XSmall);
   selected: RowNode[] = [];
   subscriptions: Subscription[] = [];
+  tabIndex: number;
+  searchModel: string;
 
   constructor(
     private serverService: ServerService,
@@ -41,8 +46,15 @@ export class ToolbarComponent implements OnInit, OnDestroy {
     private breakpointObserver: BreakpointObserver,
     private ws: WsConnectionService,
     private selectionService: SelectionService,
-    private postsDataService: PostsService
+    private postsDataService: PostsService,
+    public homeTabsService: HomeTabsService,
+    public linkCollectorService: LinkCollectorService,
+    public eventLogService: EventLogService,
   ) {
+    this.homeTabsService.index.subscribe(e => {
+      this.tabIndex = e;
+      this.search(this.searchModel);
+    });
   }
 
   openSettings(): void {
@@ -71,7 +83,17 @@ export class ToolbarComponent implements OnInit, OnDestroy {
   }
 
   search(event) {
-    this.postsDataService.search(event);
+    switch (this.tabIndex) {
+      case 0:
+        this.postsDataService.search(event);
+        break;
+      case 1:
+        this.linkCollectorService.search(event);
+        break;
+      case 2:
+        this.eventLogService.search(event);
+        break;
+    }
   }
 
   remove() {
@@ -158,6 +180,36 @@ export class ToolbarComponent implements OnInit, OnDestroy {
     });
   }
 
+  clearLinkCollector() {
+    this.ngZone.run(() => {
+      this.httpClient.get<void>(this.serverService.baseUrl + '/grab/clear', {}).subscribe(
+        () => {
+          this.linkCollectorService.clear();
+        },
+        error => {
+          this._snackBar.open(error?.error?.message || 'Unexpected error, check log file', null, {
+            duration: 5000
+          });
+        }
+      );
+    });
+  }
+
+  clearEventLogs() {
+    this.ngZone.run(() => {
+      this.httpClient.get<void>(this.serverService.baseUrl + '/events/clear', {}).subscribe(
+        () => {
+          this.eventLogService.clear();
+        },
+        error => {
+          this._snackBar.open(error?.error?.message || 'Unexpected error, check log file', null, {
+            duration: 5000
+          });
+        }
+      );
+    });
+  }
+
   stopAll() {
     this.ngZone.run(() => {
       this.httpClient.post(this.serverService.baseUrl + '/post/stop/all', {}).subscribe(

vripper-ui/src/environments/environment.prod.ts

@@ -2,5 +2,5 @@ export const environment = {
   production: true,
   localhost: `${window.location.protocol}//${window.location.host}`,
   ws: `${window.location.protocol === 'http:' ? 'ws:' : 'wss:'}//${window.location.host}`,
-  version: '3.2.3'
+  version: '3.3.0'
 };

vripper-ui/src/environments/environment.ts

@@ -6,7 +6,7 @@ export const environment = {
   production: false,
   localhost: 'http://localhost:8080',
   ws: 'ws://localhost:8080',
-  version: '3.2.3'
+  version: '3.3.0'
 };
 
 /*