diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/EventListenerBean.java b/vripper-server/src/main/java/tn/mnlr/vripper/EventListenerBean.java index c17beae..f6cdc13 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/EventListenerBean.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/EventListenerBean.java @@ -27,10 +27,7 @@ public class EventListenerBean { @EventListener public void onApplicationEvent(ContextRefreshedEvent event) { - repositorySet.forEach(IRepository::init); dataService.setDownloadingToStopped(); init = true; } - - -} \ No newline at end of file +} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/Utils.java b/vripper-server/src/main/java/tn/mnlr/vripper/Utils.java new file mode 100644 index 0000000..58b6688 --- /dev/null +++ b/vripper-server/src/main/java/tn/mnlr/vripper/Utils.java @@ -0,0 +1,17 @@ +package tn.mnlr.vripper; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; + +public class Utils { + + public static String throwableToString(Throwable th) throws IOException { + try (StringWriter stringWriter = new StringWriter(); PrintWriter printWriter = new PrintWriter(stringWriter)) { + th.printStackTrace(printWriter); + printWriter.flush(); + stringWriter.flush(); + return stringWriter.toString(); + } + } +} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadJob.java b/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadJob.java index 1e9ce38..de15458 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadJob.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadJob.java @@ -45,7 +45,7 @@ public class DownloadJob implements CheckedRunnable { private final Image image; @Getter private final Post post; - private boolean stopped = false; + private volatile boolean stopped = false; @Getter private boolean finished = false; diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadRunnable.java b/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadRunnable.java index 22b181c..9ebb0ed 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadRunnable.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/download/DownloadRunnable.java @@ -3,16 +3,22 @@ package tn.mnlr.vripper.download; import lombok.extern.slf4j.Slf4j; import net.jodah.failsafe.Failsafe; import tn.mnlr.vripper.SpringContext; +import tn.mnlr.vripper.Utils; +import tn.mnlr.vripper.jpa.domain.Event; import tn.mnlr.vripper.jpa.domain.enums.Status; +import tn.mnlr.vripper.jpa.repositories.IEventRepository; import tn.mnlr.vripper.services.ConnectionService; import tn.mnlr.vripper.services.DataService; +import java.time.LocalDateTime; + @Slf4j public class DownloadRunnable implements Runnable { private final DownloadService downloadService; private final DataService dataService; private final ConnectionService connectionService; + private final IEventRepository eventRepository; private final DownloadJob downloadJob; @@ -20,6 +26,8 @@ public class DownloadRunnable implements Runnable { downloadService = SpringContext.getBean(DownloadService.class); dataService = SpringContext.getBean(DataService.class); connectionService = SpringContext.getBean(ConnectionService.class); + eventRepository = SpringContext.getBean(IEventRepository.class); + this.downloadJob = downloadJob; } @@ -27,6 +35,12 @@ public class DownloadRunnable implements Runnable { public void run() { Failsafe.with(connectionService.getRetryPolicy()) .onFailure(e -> { + try { + Event event = new Event(Event.Type.DOWNLOAD, Event.Status.ERROR, LocalDateTime.now(), String.format("Failed to download %s\n %s", downloadJob.getImage().getUrl(), Utils.throwableToString(e.getFailure()))); + eventRepository.save(event); + } catch (Exception exp) { + log.error("Failed to save event", exp); + } log.error(String.format("Failed to download %s after %d tries", downloadJob.getImage().getUrl(), e.getAttemptCount()), e.getFailure()); downloadJob.getImage().setStatus(Status.ERROR); dataService.updateImageStatus(downloadJob.getImage().getStatus(), downloadJob.getImage().getId()); diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/event/EventRemoveEvent.java b/vripper-server/src/main/java/tn/mnlr/vripper/event/EventRemoveEvent.java new file mode 100644 index 0000000..b356826 --- /dev/null +++ b/vripper-server/src/main/java/tn/mnlr/vripper/event/EventRemoveEvent.java @@ -0,0 +1,15 @@ +package tn.mnlr.vripper.event; + +import lombok.Getter; +import org.springframework.context.ApplicationEvent; + +@Getter +public class EventRemoveEvent extends ApplicationEvent { + + private final Long id; + + public EventRemoveEvent(Object source, Long id) { + super(source); + this.id = id; + } +} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/event/EventUpdateEvent.java b/vripper-server/src/main/java/tn/mnlr/vripper/event/EventUpdateEvent.java new file mode 100644 index 0000000..78d47a4 --- /dev/null +++ b/vripper-server/src/main/java/tn/mnlr/vripper/event/EventUpdateEvent.java @@ -0,0 +1,15 @@ +package tn.mnlr.vripper.event; + +import lombok.Getter; +import org.springframework.context.ApplicationEvent; + +@Getter +public class EventUpdateEvent extends ApplicationEvent { + + private final Long id; + + public EventUpdateEvent(Object source, Long id) { + super(source); + this.id = id; + } +} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/event/MetadataUpdateEvent.java b/vripper-server/src/main/java/tn/mnlr/vripper/event/MetadataUpdateEvent.java index 8ee4fa5..fa3ec96 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/event/MetadataUpdateEvent.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/event/MetadataUpdateEvent.java @@ -6,12 +6,10 @@ import org.springframework.context.ApplicationEvent; @Getter public class MetadataUpdateEvent extends ApplicationEvent { - private final Long id; private final Long postIdRef; - public MetadataUpdateEvent(Object source, Long id, Long postIdRef) { + public MetadataUpdateEvent(Object source, Long postIdRef) { super(source); - this.id = id; this.postIdRef = postIdRef; } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/host/ImxHost.java b/vripper-server/src/main/java/tn/mnlr/vripper/host/ImxHost.java index 8ae6c58..bd76752 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/host/ImxHost.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/host/ImxHost.java @@ -68,6 +68,9 @@ public class ImxHost extends Host { try { log.debug(String.format("Looking for xpath expression %s in %s", CONTINUE_BUTTON_XPATH, url)); contDiv = xpathService.getAsNode(doc, CONTINUE_BUTTON_XPATH); + if (contDiv == null) { + throw new HostException(CONTINUE_BUTTON_XPATH + " cannot be found"); + } Node node = contDiv.getAttributes().getNamedItem("value"); if (node != null) { value = node.getTextContent(); diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/domain/Event.java b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/domain/Event.java new file mode 100644 index 0000000..86f62bb --- /dev/null +++ b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/domain/Event.java @@ -0,0 +1,64 @@ +package tn.mnlr.vripper.jpa.domain; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +@Getter +@Setter +@ToString +@NoArgsConstructor +public class Event { + + private Long id; + private Type type; + private Status status; + @JsonSerialize(using = DateTimeSerializer.class) + private LocalDateTime time; + private String message; + + public Event(Type type, Status status, LocalDateTime time, String message) { + this.type = type; + this.status = status; + this.time = time; + this.message = message; + } + + public enum Type { + POST, + QUEUED, + THANKS, + METADATA, + SCAN, + DOWNLOAD, + METADATA_CACHE_MISS, + QUEUED_CACHE_MISS + } + + public enum Status { + PENDING, PROCESSING, DONE, ERROR + } +} + +class DateTimeSerializer extends StdSerializer { + + private final static DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss.SSS"); + + protected DateTimeSerializer() { + super(LocalDateTime.class); + } + + @Override + public void serialize(LocalDateTime value, JsonGenerator gen, SerializerProvider provider) throws IOException { + gen.writeString(value.format(DATE_TIME_FORMATTER)); + } +} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/domain/Metadata.java b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/domain/Metadata.java index 8c96579..a9c8831 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/domain/Metadata.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/domain/Metadata.java @@ -1,6 +1,5 @@ package tn.mnlr.vripper.jpa.domain; -import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -13,12 +12,11 @@ import java.util.List; @NoArgsConstructor public class Metadata { - @JsonIgnore - private Long id; + private Long postIdRef; - private List resolvedNames = Collections.emptyList(); + private String PostId; private String postedBy; - private Long postIdRef; + private List resolvedNames = Collections.emptyList(); } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IEventRepository.java b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IEventRepository.java new file mode 100644 index 0000000..efa491d --- /dev/null +++ b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IEventRepository.java @@ -0,0 +1,21 @@ +package tn.mnlr.vripper.jpa.repositories; + +import tn.mnlr.vripper.jpa.domain.Event; + +import java.util.List; +import java.util.Optional; + +public interface IEventRepository extends IRepository { + + Event save(Event event); + + Event update(Event event); + + List findAll(); + + Optional findById(Long id); + + void delete(Long id); + + void deleteAll(); +} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IMetadataRepository.java b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IMetadataRepository.java index 0a20433..74c634c 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IMetadataRepository.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IMetadataRepository.java @@ -8,8 +8,6 @@ public interface IMetadataRepository extends IRepository { Metadata save(Metadata metadata); - Optional findById(Long id); - Optional findByPostId(String postId); int deleteByPostId(String postId); diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IQueuedRepository.java b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IQueuedRepository.java index 824a91a..7c9443f 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IQueuedRepository.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IQueuedRepository.java @@ -15,4 +15,6 @@ public interface IQueuedRepository extends IRepository { Optional findById(Long id); int deleteByThreadId(String threadId); + + void deleteAll(); } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IRepository.java b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IRepository.java index 0fb5fa0..89f05aa 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IRepository.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/IRepository.java @@ -1,6 +1,4 @@ package tn.mnlr.vripper.jpa.repositories; public interface IRepository { - - void init(); } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/EventRepository.java b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/EventRepository.java new file mode 100644 index 0000000..2f142a0 --- /dev/null +++ b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/EventRepository.java @@ -0,0 +1,117 @@ +package tn.mnlr.vripper.jpa.repositories.impl; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.ApplicationEventPublisherAware; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Service; +import tn.mnlr.vripper.event.EventRemoveEvent; +import tn.mnlr.vripper.event.EventUpdateEvent; +import tn.mnlr.vripper.jpa.domain.Event; +import tn.mnlr.vripper.jpa.repositories.IEventRepository; +import tn.mnlr.vripper.services.SettingsService; + +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Optional; + +@Service +@Slf4j +public class EventRepository implements IEventRepository, ApplicationEventPublisherAware { + + private final JdbcTemplate jdbcTemplate; + private final SettingsService settingsService; + private ApplicationEventPublisher applicationEventPublisher; + + public EventRepository(JdbcTemplate jdbcTemplate, SettingsService settingsService) { + this.jdbcTemplate = jdbcTemplate; + this.settingsService = settingsService; + } + + private synchronized Long nextId() { + return jdbcTemplate.queryForObject( + "CALL NEXT VALUE FOR SEQ_EVENT", + Long.class); + } + + @Override + public synchronized Event save(@NonNull Event event) { + + int maxRecords = settingsService.getSettings().getMaxEventLog() - 1; + + Long count = jdbcTemplate.queryForObject("SELECT COUNT(*) FROM EVENT", Long.class); + if (count > maxRecords) { + List idList = jdbcTemplate.queryForList("SELECT ID FROM EVENT ORDER BY TIME ASC LIMIT ?", Long.class, count - maxRecords); + idList.forEach(this::delete); + } + + long id = nextId(); + jdbcTemplate.update( + "INSERT INTO EVENT (ID, TYPE, STATUS, TIME, MESSAGE) VALUES (?,?,?,?,?)", + id, + event.getType().name(), + event.getStatus().name(), + event.getTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME), + event.getMessage() + ); + event.setId(id); + applicationEventPublisher.publishEvent(new EventUpdateEvent(EventRepository.class, id)); + return event; + } + + @Override + public Event update(@NonNull Event event) { + if (event.getId() == null) { + log.warn("Cannot update entity with null id"); + return event; + } + + jdbcTemplate.update( + "UPDATE EVENT SET TYPE = ?, STATUS = ?, TIME = ?, MESSAGE = ? WHERE ID = ?", + event.getType().name(), + event.getStatus().name(), + event.getTime().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME), + event.getMessage(), + event.getId() + ); + applicationEventPublisher.publishEvent(new EventUpdateEvent(EventRepository.class, event.getId())); + return event; + } + + @Override + public Optional findById(Long id) { + List events = jdbcTemplate.query( + "SELECT * FROM EVENT WHERE ID = ?", + new EventRowMapper(), + id); + if (events.isEmpty()) { + return Optional.empty(); + } else { + return Optional.of(events.get(0)); + } + } + + @Override + public List findAll() { + return jdbcTemplate.query( + "SELECT * FROM EVENT", + new EventRowMapper()); + } + + @Override + public void delete(Long id) { + jdbcTemplate.update("DELETE FROM EVENT WHERE ID = ?", id); + applicationEventPublisher.publishEvent(new EventRemoveEvent(EventRepository.class, id)); + } + + @Override + public void deleteAll() { + jdbcTemplate.update("DELETE FROM EVENT"); + } + + @Override + public void setApplicationEventPublisher(@NonNull ApplicationEventPublisher applicationEventPublisher) { + this.applicationEventPublisher = applicationEventPublisher; + } +} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/EventRowMapper.java b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/EventRowMapper.java new file mode 100644 index 0000000..e04a97c --- /dev/null +++ b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/EventRowMapper.java @@ -0,0 +1,23 @@ +package tn.mnlr.vripper.jpa.repositories.impl; + +import org.springframework.jdbc.core.RowMapper; +import tn.mnlr.vripper.jpa.domain.Event; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +public class EventRowMapper implements RowMapper { + + @Override + public Event mapRow(ResultSet rs, int rowNum) throws SQLException { + Event event = new Event(); + event.setId(rs.getLong("ID")); + event.setType(Event.Type.valueOf(rs.getString("TYPE"))); + event.setStatus(Event.Status.valueOf(rs.getString("STATUS"))); + event.setTime(LocalDateTime.parse(rs.getString("TIME"), DateTimeFormatter.ISO_LOCAL_DATE_TIME)); + event.setMessage(rs.getString("MESSAGE")); + return event; + } +} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/ImageRepository.java b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/ImageRepository.java index 9133c35..e68f85d 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/ImageRepository.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/ImageRepository.java @@ -4,6 +4,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.lang.NonNull; import org.springframework.stereotype.Service; import tn.mnlr.vripper.event.ImageUpdateEvent; import tn.mnlr.vripper.jpa.domain.Image; @@ -12,13 +13,11 @@ import tn.mnlr.vripper.jpa.repositories.IImageRepository; import java.util.List; import java.util.Optional; -import java.util.concurrent.atomic.AtomicLong; @Service public class ImageRepository implements IImageRepository, ApplicationEventPublisherAware { private final JdbcTemplate jdbcTemplate; - private final AtomicLong counter = new AtomicLong(0); private ApplicationEventPublisher applicationEventPublisher; @Autowired @@ -26,21 +25,15 @@ public class ImageRepository implements IImageRepository, ApplicationEventPublis this.jdbcTemplate = jdbcTemplate; } - @Override - public void init() { - Long maxId = jdbcTemplate.queryForObject( - "SELECT MAX(ID) FROM IMAGE", - Long.class - ); - if (maxId == null) { - maxId = 0L; - } - counter.set(maxId); + private synchronized Long nextId() { + return jdbcTemplate.queryForObject( + "CALL NEXT VALUE FOR SEQ_IMAGE", + Long.class); } @Override public Image save(Image image) { - long id = counter.incrementAndGet(); + long id = nextId(); jdbcTemplate.update( "INSERT INTO IMAGE (ID, CURRENT, HOST, INDEX, POST_ID, STATUS, TOTAL, URL, POST_ID_REF) VALUES (?,?,?,?,?,?,?,?,?)", id, @@ -147,7 +140,7 @@ public class ImageRepository implements IImageRepository, ApplicationEventPublis } @Override - public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) { + public void setApplicationEventPublisher(@NonNull ApplicationEventPublisher applicationEventPublisher) { this.applicationEventPublisher = applicationEventPublisher; } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/MetadataRepository.java b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/MetadataRepository.java index 2d2552d..c3b902e 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/MetadataRepository.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/MetadataRepository.java @@ -11,13 +11,11 @@ import tn.mnlr.vripper.jpa.repositories.IMetadataRepository; import java.util.List; import java.util.Optional; -import java.util.concurrent.atomic.AtomicLong; @Service public class MetadataRepository implements IMetadataRepository, ApplicationEventPublisherAware { private final JdbcTemplate jdbcTemplate; - private final AtomicLong counter = new AtomicLong(0); private ApplicationEventPublisher applicationEventPublisher; @Autowired @@ -25,53 +23,25 @@ public class MetadataRepository implements IMetadataRepository, ApplicationEvent this.jdbcTemplate = jdbcTemplate; } - @Override - public void init() { - Long maxId = jdbcTemplate.queryForObject( - "SELECT MAX(ID) FROM IMAGE", - Long.class - ); - if (maxId == null) { - maxId = 0L; - } - counter.set(maxId); - } - @Override public Metadata save(Metadata metadata) { - long id = counter.incrementAndGet(); jdbcTemplate.update( - "INSERT INTO METADATA (ID, POSTED_BY, RESOLVED_NAMES, POST_ID_REF) VALUES (?,?,?,?)", - id, + "INSERT INTO METADATA (POST_ID_REF, POST_ID, POSTED_BY, RESOLVED_NAMES) VALUES (?,?,?,?)", + metadata.getPostIdRef(), + metadata.getPostId(), metadata.getPostedBy(), - String.join("%sep%", metadata.getResolvedNames()), - metadata.getPostIdRef() + String.join("%sep%", metadata.getResolvedNames()) ); - metadata.setId(id); - applicationEventPublisher.publishEvent(new MetadataUpdateEvent(MetadataRepository.class, id, metadata.getPostIdRef())); + applicationEventPublisher.publishEvent(new MetadataUpdateEvent(MetadataRepository.class, metadata.getPostIdRef())); return metadata; } - @Override - public Optional findById(Long id) { - List metadata = jdbcTemplate.query( - "SELECT * FROM METADATA AS metadata WHERE metadata.ID = ?", - new Object[]{id}, - new MetadataRowMapper() - ); - if (metadata.isEmpty()) { - return Optional.empty(); - } else { - return Optional.of(metadata.get(0)); - } - } - @Override public Optional findByPostId(String postId) { List metadata = jdbcTemplate.query( - "SELECT metadata.* FROM METADATA AS metadata INNER JOIN POST post ON post.ID = metadata.POST_ID_REF WHERE post.POST_ID = ?", - new Object[]{postId}, - new MetadataRowMapper() + "SELECT metadata.* FROM METADATA AS metadata WHERE metadata.POST_ID = ?", + new MetadataRowMapper(), + postId ); if (metadata.isEmpty()) { return Optional.empty(); @@ -83,7 +53,7 @@ public class MetadataRepository implements IMetadataRepository, ApplicationEvent @Override public int deleteByPostId(String postId) { return jdbcTemplate.update( - "DELETE FROM METADATA AS metadata WHERE metadata.ID = (SELECT inner_metadata.ID FROM POST AS post INNER JOIN METADATA inner_metadata ON post.ID = inner_metadata.POST_ID_REF WHERE post.POST_ID = ?)", + "DELETE FROM METADATA AS metadata WHERE metadata.POST_ID = ?", postId ); } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/MetadataRowMapper.java b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/MetadataRowMapper.java index 29833dc..0b22a36 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/MetadataRowMapper.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/MetadataRowMapper.java @@ -12,13 +12,13 @@ public class MetadataRowMapper implements RowMapper { @Override public Metadata mapRow(ResultSet rs, int rowNum) throws SQLException { Metadata metadata = new Metadata(); - metadata.setId(rs.getLong("ID")); + metadata.setPostIdRef(rs.getLong("POST_ID_REF")); + metadata.setPostId(rs.getString("POST_ID")); metadata.setPostedBy(rs.getString("POSTED_BY")); String resolvedNames = rs.getString("RESOLVED_NAMES"); if (resolvedNames != null && !resolvedNames.isBlank()) { metadata.setResolvedNames(List.of(resolvedNames.split("%sep%"))); } - metadata.setPostIdRef(rs.getLong("POST_ID_REF")); return metadata; } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/PostRepository.java b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/PostRepository.java index 5f3b517..bbaa191 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/PostRepository.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/PostRepository.java @@ -4,6 +4,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.lang.NonNull; import org.springframework.stereotype.Service; import tn.mnlr.vripper.event.PostRemoveEvent; import tn.mnlr.vripper.event.PostUpdateEvent; @@ -13,13 +14,11 @@ import tn.mnlr.vripper.jpa.repositories.IPostRepository; import java.util.List; import java.util.Optional; -import java.util.concurrent.atomic.AtomicLong; @Service public class PostRepository implements IPostRepository, ApplicationEventPublisherAware { private final JdbcTemplate jdbcTemplate; - private final AtomicLong counter = new AtomicLong(0); private ApplicationEventPublisher applicationEventPublisher; @Autowired @@ -28,21 +27,15 @@ public class PostRepository implements IPostRepository, ApplicationEventPublishe } - @Override - public void init() { - Long maxId = jdbcTemplate.queryForObject( - "SELECT MAX(ID) FROM POST", - Long.class - ); - if (maxId == null) { - maxId = 0L; - } - counter.set(maxId); + private synchronized Long nextId() { + return jdbcTemplate.queryForObject( + "CALL NEXT VALUE FOR SEQ_POST", + Long.class); } @Override public Post save(Post post) { - long id = counter.incrementAndGet(); + long id = nextId(); jdbcTemplate.update( "INSERT INTO POST (ID, DONE, FORUM, HOSTS, POST_FOLDER_NAME, POST_ID, PREVIEWS, SECURITY_TOKEN, STATUS, THANKED, THREAD_ID, THREAD_TITLE, TITLE, TOTAL, URL) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", id, @@ -69,7 +62,7 @@ public class PostRepository implements IPostRepository, ApplicationEventPublishe @Override public Optional findByPostId(String postId) { List posts = jdbcTemplate.query( - "SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE POST_ID = ?", + "SELECT metadata.*,post.* FROM METADATA metadata FULL JOIN POST post ON metadata.POST_ID_REF = post.ID WHERE post.POST_ID = ?", new PostRowMapper(), postId); if (posts.isEmpty()) { @@ -82,7 +75,7 @@ public class PostRepository implements IPostRepository, ApplicationEventPublishe @Override public List findCompleted() { return jdbcTemplate.query( - "SELECT POST_ID FROM POST WHERE status = 'COMPLETE' AND done >= total", + "SELECT POST_ID FROM POST AS post WHERE status = 'COMPLETE' AND done >= total", ((rs, rowNum) -> rs.getString("POST_ID")) ); } @@ -189,7 +182,7 @@ public class PostRepository implements IPostRepository, ApplicationEventPublishe } @Override - public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) { + public void setApplicationEventPublisher(@NonNull ApplicationEventPublisher applicationEventPublisher) { this.applicationEventPublisher = applicationEventPublisher; } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/PostRowMapper.java b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/PostRowMapper.java index 1ad5166..e3a3ebb 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/PostRowMapper.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/PostRowMapper.java @@ -19,34 +19,34 @@ public class PostRowMapper implements RowMapper { Post post = new Post(); post.setId(rs.getLong("post.ID")); - post.setStatus(Status.valueOf(rs.getString("STATUS"))); - post.setPostId(rs.getString("POST_ID")); - post.setThreadTitle(rs.getString("THREAD_TITLE")); - post.setThreadId(rs.getString("THREAD_ID")); - post.setTitle(rs.getString("TITLE")); - post.setUrl(rs.getString("URL")); - post.setDone(rs.getInt("DONE")); - post.setTotal(rs.getInt("TOTAL")); - post.setHosts(Set.of(rs.getString("HOSTS").split(DELIMITER))); - post.setForum(rs.getString("FORUM")); - post.setSecurityToken(rs.getString("SECURITY_TOKEN")); - post.setDownloadDirectory(rs.getString("POST_FOLDER_NAME")); - post.setThanked(rs.getBoolean("THANKED")); + post.setStatus(Status.valueOf(rs.getString("post.STATUS"))); + post.setPostId(rs.getString("post.POST_ID")); + post.setThreadTitle(rs.getString("post.THREAD_TITLE")); + post.setThreadId(rs.getString("post.THREAD_ID")); + post.setTitle(rs.getString("post.TITLE")); + post.setUrl(rs.getString("post.URL")); + post.setDone(rs.getInt("post.DONE")); + post.setTotal(rs.getInt("post.TOTAL")); + post.setHosts(Set.of(rs.getString("post.HOSTS").split(DELIMITER))); + post.setForum(rs.getString("post.FORUM")); + post.setSecurityToken(rs.getString("post.SECURITY_TOKEN")); + post.setDownloadDirectory(rs.getString("post.POST_FOLDER_NAME")); + post.setThanked(rs.getBoolean("post.THANKED")); String previews; - if ((previews = rs.getString("PREVIEWS")) != null) { + if ((previews = rs.getString("post.PREVIEWS")) != null) { post.setPreviews(Set.of(previews.split(DELIMITER))); } - Long metadataId = rs.getLong("metadata.ID"); + Long metadataId = rs.getLong("metadata.POST_ID_REF"); if (!rs.wasNull()) { Metadata metadata = new Metadata(); - metadata.setId(metadataId); - metadata.setPostIdRef(rs.getLong("POST_ID_REF")); - String resolvedNames = rs.getString("RESOLVED_NAMES"); + metadata.setPostIdRef(metadataId); + metadata.setPostId(rs.getString("metadata.POST_ID")); + String resolvedNames = rs.getString("metadata.RESOLVED_NAMES"); if (resolvedNames != null && !resolvedNames.isBlank()) { metadata.setResolvedNames(List.of(resolvedNames.split("%sep%"))); } - metadata.setPostedBy(rs.getString("POSTED_BY")); + metadata.setPostedBy(rs.getString("metadata.POSTED_BY")); post.setMetadata(metadata); } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/QueuedRepository.java b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/QueuedRepository.java index 656ad0c..15ddf93 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/QueuedRepository.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/jpa/repositories/impl/QueuedRepository.java @@ -12,13 +12,11 @@ import tn.mnlr.vripper.jpa.repositories.IQueuedRepository; import java.util.List; import java.util.Optional; -import java.util.concurrent.atomic.AtomicLong; @Service public class QueuedRepository implements IQueuedRepository, ApplicationEventPublisherAware { private final JdbcTemplate jdbcTemplate; - private final AtomicLong counter = new AtomicLong(0); private ApplicationEventPublisher applicationEventPublisher; @Autowired @@ -26,21 +24,15 @@ public class QueuedRepository implements IQueuedRepository, ApplicationEventPubl this.jdbcTemplate = jdbcTemplate; } - @Override - public void init() { - Long maxId = jdbcTemplate.queryForObject( - "SELECT MAX(ID) FROM QUEUED", - Long.class - ); - if (maxId == null) { - maxId = 0L; - } - counter.set(maxId); + private synchronized Long nextId() { + return jdbcTemplate.queryForObject( + "CALL NEXT VALUE FOR SEQ_QUEUED", + Long.class); } @Override public Queued save(Queued queued) { - long id = counter.incrementAndGet(); + long id = nextId(); jdbcTemplate.update( "INSERT INTO QUEUED (ID, TOTAL, LINK, LOADING, POST_ID, THREAD_ID) values (?,?,?,?,?,?)", id, @@ -101,6 +93,11 @@ public class QueuedRepository implements IQueuedRepository, ApplicationEventPubl return mutationCount; } + @Override + public void deleteAll() { + jdbcTemplate.update("DELETE FROM QUEUED"); + } + @Override public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) { this.applicationEventPublisher = applicationEventPublisher; diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/listener/EventRemoveEventListener.java b/vripper-server/src/main/java/tn/mnlr/vripper/listener/EventRemoveEventListener.java new file mode 100644 index 0000000..5318b3a --- /dev/null +++ b/vripper-server/src/main/java/tn/mnlr/vripper/listener/EventRemoveEventListener.java @@ -0,0 +1,32 @@ +package tn.mnlr.vripper.listener; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationListener; +import org.springframework.stereotype.Component; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Sinks; +import tn.mnlr.vripper.event.EventRemoveEvent; + +import javax.annotation.PreDestroy; + +@Component +@Slf4j +public class EventRemoveEventListener implements ApplicationListener, DataEventListener { + + private final Sinks.Many sink = Sinks.many().multicast().onBackpressureBuffer(); + + @Override + public void onApplicationEvent(EventRemoveEvent event) { + sink.emitNext(event, EmitHandler.RETRY); + } + + @Override + public Flux getDataFlux() { + return sink.asFlux(); + } + + @PreDestroy + private void destroy() { + sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST); + } +} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/listener/EventUpdateEventListener.java b/vripper-server/src/main/java/tn/mnlr/vripper/listener/EventUpdateEventListener.java new file mode 100644 index 0000000..f12bdab --- /dev/null +++ b/vripper-server/src/main/java/tn/mnlr/vripper/listener/EventUpdateEventListener.java @@ -0,0 +1,33 @@ +package tn.mnlr.vripper.listener; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationListener; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Component; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Sinks; +import tn.mnlr.vripper.event.EventUpdateEvent; + +import javax.annotation.PreDestroy; + +@Component +@Slf4j +public class EventUpdateEventListener implements ApplicationListener, DataEventListener { + + private final Sinks.Many sink = Sinks.many().multicast().onBackpressureBuffer(); + + @Override + public void onApplicationEvent(@NonNull EventUpdateEvent event) { + sink.emitNext(event, EmitHandler.RETRY); + } + + @Override + public Flux getDataFlux() { + return sink.asFlux(); + } + + @PreDestroy + private void destroy() { + sink.emitComplete(Sinks.EmitFailureHandler.FAIL_FAST); + } +} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/DataService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/DataService.java index 3fc4f4e..0de5023 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/DataService.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/DataService.java @@ -5,15 +5,13 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import tn.mnlr.vripper.jpa.domain.Image; -import tn.mnlr.vripper.jpa.domain.Metadata; -import tn.mnlr.vripper.jpa.domain.Post; -import tn.mnlr.vripper.jpa.domain.Queued; +import tn.mnlr.vripper.jpa.domain.*; import tn.mnlr.vripper.jpa.domain.enums.Status; import tn.mnlr.vripper.jpa.repositories.IImageRepository; import tn.mnlr.vripper.jpa.repositories.IMetadataRepository; import tn.mnlr.vripper.jpa.repositories.IPostRepository; import tn.mnlr.vripper.jpa.repositories.IQueuedRepository; +import tn.mnlr.vripper.jpa.repositories.impl.EventRepository; import java.util.Collection; import java.util.List; @@ -29,14 +27,16 @@ public class DataService { private final IQueuedRepository queuedRepository; private final IMetadataRepository metadataRepository; private final SettingsService settingsService; + private final EventRepository eventRepository; @Autowired - public DataService(IPostRepository postRepository, IImageRepository imageRepository, IQueuedRepository queuedRepository, IMetadataRepository metadataRepository, SettingsService settingsService) { + public DataService(IPostRepository postRepository, IImageRepository imageRepository, IQueuedRepository queuedRepository, IMetadataRepository metadataRepository, SettingsService settingsService, EventRepository eventRepository) { this.postRepository = postRepository; this.imageRepository = imageRepository; this.queuedRepository = queuedRepository; this.metadataRepository = metadataRepository; this.settingsService = settingsService; + this.eventRepository = eventRepository; } private void save(Post post) { @@ -142,7 +142,7 @@ public class DataService { return imageRepository.findByPostId(postId); } - public Iterable findAllPosts() { + public List findAllPosts() { return postRepository.findAll(); } @@ -158,7 +158,7 @@ public class DataService { return queuedRepository.findByThreadId(threadId); } - public Iterable findAllQueued() { + public List findAllQueued() { return queuedRepository.findAll(); } @@ -174,9 +174,11 @@ public class DataService { return queuedRepository.findById(aLong); } - public void setMetadata(Post post, Metadata metadata) { - metadata.setPostIdRef(post.getId()); - metadataRepository.save(metadata); + public synchronized void setMetadata(Post post, Metadata metadata) { + if (metadataRepository.findByPostId(post.getPostId()).isEmpty()) { + metadata.setPostIdRef(post.getId()); + metadataRepository.save(metadata); + } } public Optional findMetadataByPostId(String postId) { @@ -210,4 +212,16 @@ public class DataService { public void updatePostThanked(boolean thanked, Long id) { postRepository.updateThanked(thanked, id); } + + public Optional findEventById(Long id) { + return eventRepository.findById(id); + } + + public List findAllEvents() { + return eventRepository.findAll(); + } + + public void clearQueueLinks() { + queuedRepository.deleteAll(); + } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/MetadataService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/MetadataService.java index 1a078f2..379ee0b 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/MetadataService.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/MetadataService.java @@ -4,7 +4,6 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import lombok.Getter; -import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import net.jodah.failsafe.Failsafe; import org.apache.http.client.HttpClient; @@ -12,18 +11,21 @@ import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.util.EntityUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.lang.NonNull; import org.springframework.stereotype.Service; import org.w3c.dom.Document; import org.w3c.dom.Node; import tn.mnlr.vripper.exception.DownloadException; import tn.mnlr.vripper.exception.PostParseException; +import tn.mnlr.vripper.jpa.domain.Event; import tn.mnlr.vripper.jpa.domain.Metadata; import tn.mnlr.vripper.jpa.domain.Post; +import tn.mnlr.vripper.jpa.repositories.IEventRepository; +import tn.mnlr.vripper.services.domain.tasks.MetadataRunnable; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; +import java.time.LocalDateTime; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -39,12 +41,18 @@ public class MetadataService { private final VGAuthService VGAuthService; private final HtmlProcessorService htmlProcessorService; private final XpathService xpathService; + private final Map fetchingMetadata = new ConcurrentHashMap<>(); + private final ThreadPoolService threadPoolService; + private final IEventRepository eventRepository; + @Autowired - public MetadataService(ConnectionService cm, VGAuthService VGAuthService, HtmlProcessorService htmlProcessorService, XpathService xpathService) { + public MetadataService(ConnectionService cm, VGAuthService VGAuthService, HtmlProcessorService htmlProcessorService, XpathService xpathService, ThreadPoolService threadPoolService, IEventRepository eventRepository) { this.cm = cm; this.VGAuthService = VGAuthService; this.htmlProcessorService = htmlProcessorService; this.xpathService = xpathService; + this.threadPoolService = threadPoolService; + this.eventRepository = eventRepository; CacheLoader loader = new CacheLoader<>() { @Override @@ -57,14 +65,47 @@ public class MetadataService { .build(loader); } - public Metadata get(Post post) throws ExecutionException { + public Metadata get(Post post) { Metadata metadata = new Metadata(); - Metadata cachedMetadata = cache.get(new Key(post.getPostId(), post.getThreadId(), post.getUrl())); + Key key = new Key(post.getPostId(), post.getThreadId(), post.getUrl()); + Metadata cachedMetadata = cache.getIfPresent(key); + if (cachedMetadata == null) { + Event event = new Event(Event.Type.METADATA_CACHE_MISS, Event.Status.PROCESSING, LocalDateTime.now(), "Loading metadata for " + post.getUrl()); + eventRepository.save(event); + try { + cachedMetadata = cache.get(key); + event.setStatus(Event.Status.DONE); + eventRepository.update(event); + } catch (ExecutionException e) { + String error = "Failed to load metadata for " + post.getUrl(); + log.error(error, e); + event.setStatus(Event.Status.ERROR); + event.setMessage(error + ": " + e.getMessage()); + eventRepository.update(event); + return null; + } + } metadata.setPostedBy(cachedMetadata.getPostedBy()); + metadata.setPostId(cachedMetadata.getPostId()); metadata.setResolvedNames(List.copyOf(cachedMetadata.getResolvedNames())); return metadata; } + public void startFetchingMetadata(Post post) { + MetadataRunnable runnable = new MetadataRunnable(post); + threadPoolService.getGeneralExecutor().submit(runnable); + fetchingMetadata.put(post.getPostId(), runnable); + } + + public void stopFetchingMetadata(Post post) { + this.fetchingMetadata.forEach((k, v) -> { + if (k.equals(post.getPostId())) { + v.setInterrupted(true); + } + }); + fetchingMetadata.remove(post.getPostId()); + } + private Metadata fetchMetadata(Key key) { HttpGet httpGet = cm.buildHttpGet(key.getUrl(), null); Metadata metadata = new Metadata(); @@ -93,6 +134,8 @@ public class MetadataService { Node node = xpathService.getAsNode(document, String.format("//div[@id='post_message_%s']", key.getPostId())); metadata.setResolvedNames(findTitleInContent(node)); + + metadata.setPostId(key.getPostId()); } catch (Exception e) { throw new PostParseException(String.format("Failed to parse thread %s, post %s", key.getThreadId(), key.getPostId()), e); } finally { diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/PostService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/PostService.java index 750105c..240d81f 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/PostService.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/PostService.java @@ -3,48 +3,43 @@ package tn.mnlr.vripper.services; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import lombok.Getter; -import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.lang.NonNull; import org.springframework.stereotype.Service; -import tn.mnlr.vripper.download.PendingQueue; -import tn.mnlr.vripper.exception.PostParseException; -import tn.mnlr.vripper.jpa.domain.Image; +import tn.mnlr.vripper.jpa.domain.Event; import tn.mnlr.vripper.jpa.domain.Post; import tn.mnlr.vripper.jpa.domain.Queued; -import tn.mnlr.vripper.jpa.domain.enums.Status; -import tn.mnlr.vripper.services.domain.*; +import tn.mnlr.vripper.jpa.repositories.IEventRepository; +import tn.mnlr.vripper.services.domain.ApiThreadParser; +import tn.mnlr.vripper.services.domain.MultiPostItem; +import tn.mnlr.vripper.services.domain.tasks.AddPostRunnable; +import tn.mnlr.vripper.services.domain.tasks.AddQueuedRunnable; +import java.time.LocalDateTime; import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import static tn.mnlr.vripper.jpa.domain.Event.Status.DONE; +import static tn.mnlr.vripper.jpa.domain.Event.Status.PROCESSING; + @Service @Slf4j public class PostService { - private final SettingsService settingsService; - private final PendingQueue pendingQueue; private final DataService dataService; private final ThreadPoolService threadPoolService; - private final Map> fetchingMetadata = new ConcurrentHashMap<>(); - private final VGAuthService VGAuthService; - - @Getter + private final MetadataService metadataService; + private final IEventRepository eventRepository; private final LoadingCache> cache; @Autowired - public PostService(SettingsService settingsService, PendingQueue pendingQueue, DataService dataService, ThreadPoolService threadPoolService, VGAuthService VGAuthService) { - this.settingsService = settingsService; - this.pendingQueue = pendingQueue; + public PostService(DataService dataService, ThreadPoolService threadPoolService, MetadataService metadataService, IEventRepository eventRepository) { this.dataService = dataService; this.threadPoolService = threadPoolService; - this.VGAuthService = VGAuthService; + this.metadataService = metadataService; + this.eventRepository = eventRepository; CacheLoader> loader = new CacheLoader<>() { @Override @@ -59,95 +54,41 @@ public class PostService { .build(loader); } - public void addPost(String postId, String threadId) throws PostParseException { - - if (dataService.exists(postId)) { - log.warn(String.format("skipping %s, already loaded", postId)); - return; - } - - ApiPostParser apiPostParser = new ApiPostParser(threadId, postId); - ApiPost apiPost = apiPostParser.parse(); - if (apiPost.getPost().isEmpty()) { - throw new PostParseException(String.format("parsing failed for thread %s, post %s", threadId, postId)); - } - - Post post = apiPost.getPost().get(); - Set images = apiPost.getImages(); - - dataService.newPost(post, images); - - // Metadata thread - fetchingMetadata.put(post.getPostId(), threadPoolService.getGeneralExecutor().submit(new MetadataRunnable(post))); - - if (settingsService.getSettings().getAutoStart()) { - log.debug("Auto start downloads option is enabled"); - post.setStatus(Status.PENDING); - try { - pendingQueue.enqueue(post, images); - } catch (InterruptedException e) { - log.warn("Interruption was caught"); - Thread.currentThread().interrupt(); - return; - } - log.debug(String.format("Done enqueuing jobs for %s", post.getUrl())); - } else { - post.setStatus(Status.STOPPED); - log.debug("Auto start downloads option is disabled"); - } - if (!settingsService.getSettings().getLeaveThanksOnStart()) { - VGAuthService.leaveThanks(post); - } - dataService.updatePostStatus(post.getStatus(), post.getId()); - } - public void stopFetchingMetadata(Post post) { - this.fetchingMetadata.forEach((k, v) -> { - if (k.equals(post.getPostId())) { - v.cancel(true); - } - }); - fetchingMetadata.remove(post.getPostId()); + metadataService.stopFetchingMetadata(post); } - public void processMultiPost(List queuedList) throws Exception { + public void processMultiPost(List queuedList) { for (Queued queued : queuedList) { if (queued.getPostId() != null) { - addPost(queued.getPostId(), queued.getThreadId()); + threadPoolService.getGeneralExecutor().submit(new AddPostRunnable(queued.getPostId(), queued.getThreadId())); } else { - threadPoolService.getGeneralExecutor().submit(() -> this.multiPost(queued)); + threadPoolService.getGeneralExecutor().submit(new AddQueuedRunnable(queued)); } } } - private void multiPost(Queued queued) { - + public List get(Queued queued) { List multiPostItems; - try { - multiPostItems = cache.get(queued); - } catch (ExecutionException e) { - log.error(String.format("Failed to add post with thread id %s, postId %s", queued.getThreadId(), queued.getPostId()), e); - return; - } - queued.setTotal(multiPostItems.size()); - queued.done(); - log.debug(String.format("%d found for %s", multiPostItems.size(), queued.getLink())); - if (multiPostItems.size() == 1) { + multiPostItems = cache.getIfPresent(queued); + if (multiPostItems == null) { + Event event = new Event(Event.Type.QUEUED_CACHE_MISS, PROCESSING, LocalDateTime.now(), "Loading posts from " + queued.getLink()); + eventRepository.save(event); try { - addPost(multiPostItems.get(0).getPostId(), multiPostItems.get(0).getThreadId()); - } catch (PostParseException e) { - log.error(String.format("Failed to add post with postId %s", multiPostItems.get(0).getPostId()), e); - return; - } - log.debug(String.format("threadId %s, postId %s is added automatically for download", queued.getThreadId(), queued.getPostId())); - } else { - if (dataService.findQueuedByThreadId(queued.getThreadId()).isEmpty()) { - dataService.newQueueLink(queued); - } else { - log.info(String.format("Thread with id = %s is already loaded", queued.getThreadId())); + multiPostItems = cache.get(queued); + event.setStatus(DONE); + event.setMessage("Loaded " + multiPostItems.size() + " posts from " + queued.getLink()); + eventRepository.update(event); + } catch (ExecutionException e) { + String error = String.format("Failed to parse link %s", queued.getLink()); + log.error(error, e); + event.setStatus(Event.Status.ERROR); + event.setMessage(error + ": " + e.getMessage()); + eventRepository.update(event); + return null; } } - + return multiPostItems; } public void remove(String threadId) { diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/SettingsService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/SettingsService.java index 8d3cb16..a8caf92 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/SettingsService.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/SettingsService.java @@ -2,6 +2,7 @@ package tn.mnlr.vripper.services; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.Getter; @@ -10,6 +11,7 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.Resource; import org.springframework.stereotype.Service; import reactor.core.publisher.Flux; import reactor.core.publisher.Sinks; @@ -19,11 +21,17 @@ import tn.mnlr.vripper.listener.EmitHandler; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; +import java.io.FileWriter; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; import static java.nio.file.StandardOpenOption.*; @@ -33,28 +41,67 @@ import static java.nio.file.StandardOpenOption.*; public class SettingsService { private final Path configPath; + private final Path customProxiesPath; private final ObjectMapper om = new ObjectMapper(); + private final Set proxies = new HashSet<>(); + private Sinks.Many sink = Sinks.many().multicast().onBackpressureBuffer(); @Getter private Settings settings = new Settings(); + @Value("classpath:proxies.json") + private Resource defaultProxies; public SettingsService(@Value("${base.dir}") String baseDir, @Value("${base.dir.name}") String baseDirName) { this.configPath = Paths.get(baseDir, baseDirName, "config.json"); + this.customProxiesPath = Paths.get(baseDir, baseDirName, "proxies.json"); om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); } @PostConstruct private void init() { + loadViperProxies(); restore(); sink.emitNext(settings, EmitHandler.RETRY); } + private void loadViperProxies() { + try (InputStream defaultProxiesIs = defaultProxies.getInputStream()) { + List defaultProxies = om.readValue(defaultProxiesIs, new TypeReference<>() { + }); + List customProxies = new ArrayList<>(); + if (customProxiesPath.toFile().exists() && customProxiesPath.toFile().isFile()) { + customProxies = om.readValue(customProxiesPath.toFile(), new TypeReference<>() { + }); + } else { + if (!customProxiesPath.toFile().createNewFile()) { + log.warn("Unable to create " + customProxiesPath.toFile().getAbsolutePath()); + } else { + try (FileWriter fw = new FileWriter(customProxiesPath.toFile())) { + fw.append("[]").append(System.lineSeparator()); + fw.flush(); + } catch (IOException e) { + log.warn("Unable to create " + customProxiesPath.toFile().getAbsolutePath(), e); + } + } + } + proxies.addAll(customProxies); + proxies.addAll(defaultProxies); + } catch (IOException e) { + log.error("Failed to load vipergirls proxies list", e); + } + proxies.add("https://vipergirls.to"); + } + Flux getSettingsFlux() { return sink.asFlux(); } + public List getProxies() { + return new ArrayList<>(proxies); + } + public void newSettings(Settings settings) { if (settings.getVLogin() != null && settings.getVLogin()) { @@ -75,7 +122,9 @@ public class SettingsService { public void restore() { try { - settings = om.readValue(configPath.toFile(), Settings.class); + if (configPath.toFile().exists()) { + settings = om.readValue(configPath.toFile(), Settings.class); + } } catch (IOException e) { log.error("Failed restore user settings", e); settings = new Settings(); @@ -153,6 +202,14 @@ public class SettingsService { settings.setMaxAttempts(5); } + if (settings.getVProxy() == null || !proxies.contains(settings.getVProxy())) { + settings.setVProxy("https://vipergirls.to"); + } + + if (settings.getMaxEventLog() == null) { + settings.setMaxEventLog(1000); + } + try { check(this.settings); } catch (ValidationException e) { @@ -207,6 +264,10 @@ public class SettingsService { if (settings.getMaxAttempts() < 1 || settings.getMaxAttempts() > 10) { throw new ValidationException(String.format("Invalid maximum attempts settings, values must be in [%d,%d]", 1, 10)); } + + if (settings.getMaxEventLog() < 100 || settings.getMaxEventLog() > 10_000) { + throw new ValidationException(String.format("Invalid maximum event log record settings, values must be in [%d,%d]", 100, 10_000)); + } } public Theme getTheme() { @@ -289,5 +350,12 @@ public class SettingsService { @JsonProperty("maxAttempts") private Integer maxAttempts; + + @JsonProperty("vProxy") + private String vProxy; + + @JsonProperty("maxEventLog") + private Integer maxEventLog; + } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/VGAuthService.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/VGAuthService.java index ca075c3..e3f35b4 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/VGAuthService.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/VGAuthService.java @@ -20,6 +20,7 @@ import reactor.core.publisher.Sinks; import tn.mnlr.vripper.exception.VripperException; import tn.mnlr.vripper.jpa.domain.Post; import tn.mnlr.vripper.listener.EmitHandler; +import tn.mnlr.vripper.services.domain.tasks.LeaveThanksRunnable; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -33,7 +34,6 @@ public class VGAuthService { private final ConnectionService cm; private final SettingsService settingsService; private final ThreadPoolService threadPoolService; - private final DataService dataService; private final Disposable disposable; @@ -46,11 +46,10 @@ public class VGAuthService { private String loggedUser = ""; @Autowired - public VGAuthService(ConnectionService cm, SettingsService settingsService, ThreadPoolService threadPoolService, DataService dataService) { + public VGAuthService(ConnectionService cm, SettingsService settingsService, ThreadPoolService threadPoolService) { this.cm = cm; this.settingsService = settingsService; this.threadPoolService = threadPoolService; - this.dataService = dataService; disposable = settingsService.getSettingsFlux().subscribe(settings -> this.authenticate()); } @@ -94,7 +93,7 @@ public class VGAuthService { return; } - HttpPost postAuth = cm.buildHttpPost("https://vipergirls.to/login.php?do=login", null); + HttpPost postAuth = cm.buildHttpPost(settingsService.getSettings().getVProxy() + "/login.php?do=login", null); List params = new ArrayList<>(); params.add(new BasicNameValuePair("vb_login_username", username)); @@ -107,12 +106,12 @@ public class VGAuthService { context.getCookieStore().clear(); loggedUser = ""; sink.emitNext(loggedUser, EmitHandler.RETRY); - log.error("Failed to authenticate user with vipergirls.to", e); + log.error("Failed to authenticate user with " + settingsService.getSettings().getVProxy(), e); return; } - postAuth.addHeader("Referer", "https://vipergirls.to/"); - postAuth.addHeader("Host", "vipergirls.to"); + postAuth.addHeader("Referer", settingsService.getSettings().getVProxy()); + postAuth.addHeader("Host", settingsService.getSettings().getVProxy().replace("https://", "").replace("http://", "")); CloseableHttpClient client = cm.getClient().build(); @@ -125,14 +124,14 @@ public class VGAuthService { log.debug(String.format("Authentication with ViperGirls response body:%n%s", responseBody)); EntityUtils.consumeQuietly(response.getEntity()); if (context.getCookieStore().getCookies().stream().map(Cookie::getName).noneMatch(e -> e.equals("vg_userid"))) { - log.error("Failed to authenticate user with vipergirls.to, missing vg_userid cookie"); + log.error(String.format("Failed to authenticate user with %s, missing vg_userid cookie", settingsService.getSettings().getVProxy())); return; } } catch (Exception e) { context.getCookieStore().clear(); loggedUser = ""; sink.emitNext(loggedUser, EmitHandler.RETRY); - log.error("Failed to authenticate user with vipergirls.to", e); + log.error("Failed to authenticate user with " + settingsService.getSettings().getVProxy(), e); return; } authenticated = true; @@ -142,62 +141,6 @@ public class VGAuthService { } public void leaveThanks(Post post) { - if (!settingsService.getSettings().getVLogin()) { - log.debug("Authentication with ViperGirls option is disabled"); - return; - } - if (!settingsService.getSettings().getVThanks()) { - log.debug("Leave thanks option is disabled"); - return; - } - if (!authenticated) { - log.error("You are not authenticated"); - return; - } - if (post.isThanked()) { - log.debug("Already left a thanks"); - return; - } - threadPoolService.getGeneralExecutor().submit(() -> { - try { - postThanks(post); - } catch (Exception e) { - log.error(String.format("Failed to leave a thanks for url %s, post id %s", post.getUrl(), post.getPostId()), e); - } - }); - } - - private void postThanks(Post post) throws VripperException { - - HttpPost postThanks = cm.buildHttpPost("https://vipergirls.to/post_thanks.php", null); - List params = new ArrayList<>(); - params.add(new BasicNameValuePair("do", "post_thanks_add")); - params.add(new BasicNameValuePair("using_ajax", "1")); - params.add(new BasicNameValuePair("p", post.getPostId())); - params.add(new BasicNameValuePair("securitytoken", post.getSecurityToken())); - try { - postThanks.setEntity(new UrlEncodedFormEntity(params)); - } catch (Exception e) { - throw new VripperException(e); - } - - postThanks.addHeader("Referer", "https://vipergirls.to/"); - postThanks.addHeader("Host", "vipergirls.to"); - - CloseableHttpClient client = cm.getClient().build(); - - try (CloseableHttpResponse response = client.execute(postThanks, context)) { - if (response.getStatusLine().getStatusCode() / 100 == 2) { - post.setThanked(true); - dataService.updatePostThanked(post.isThanked(), post.getId()); - } - EntityUtils.consumeQuietly(response.getEntity()); - } catch (Exception e) { - throw new VripperException(e); - } - - if (!post.isThanked()) { - throw new VripperException("Failed to leave"); - } + threadPoolService.getGeneralExecutor().submit(new LeaveThanksRunnable(post, authenticated, context)); } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiPostHandler.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiPostHandler.java index 9db8320..483265a 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiPostHandler.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiPostHandler.java @@ -7,6 +7,7 @@ import tn.mnlr.vripper.SpringContext; import tn.mnlr.vripper.host.Host; import tn.mnlr.vripper.jpa.domain.Image; import tn.mnlr.vripper.jpa.domain.Post; +import tn.mnlr.vripper.services.SettingsService; import java.util.Collection; import java.util.HashSet; @@ -25,7 +26,6 @@ class ApiPostHandler extends DefaultHandler { private final Set images = new HashSet<>(); private Set previews = new HashSet<>(); private String threadTitle; - private String postTitle; private String forum; private String userHash; private int index = 0; @@ -35,8 +35,9 @@ class ApiPostHandler extends DefaultHandler { ApiPostHandler(String threadId, String postId) { this.threadId = threadId; this.postId = postId; - this.postUrl = String.format("https://vipergirls.to/threads/%s/?p=%s&viewfull=1#post%s", this.threadId, this.postId, this.postId); - this.supportedHosts = SpringContext.getBeansOfType(Host.class).values(); + supportedHosts = SpringContext.getBeansOfType(Host.class).values(); + SettingsService settingsService = SpringContext.getBean(SettingsService.class); + postUrl = String.format("%s/threads/%s/?p=%s&viewfull=1#post%s", settingsService.getSettings().getVProxy(), this.threadId, this.postId, this.postId); } @@ -62,7 +63,7 @@ class ApiPostHandler extends DefaultHandler { threadTitle = attributes.getValue("title").trim(); break; case "post": - postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle); + String postTitle = Optional.ofNullable(attributes.getValue("title")).map(e -> e.trim().isEmpty() ? null : e.trim()).orElse(threadTitle); parsedPost = new Post( postTitle, postUrl, diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiPostParser.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiPostParser.java index 24809b8..81961f9 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiPostParser.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiPostParser.java @@ -11,6 +11,7 @@ import tn.mnlr.vripper.SpringContext; import tn.mnlr.vripper.exception.DownloadException; import tn.mnlr.vripper.exception.PostParseException; import tn.mnlr.vripper.services.ConnectionService; +import tn.mnlr.vripper.services.SettingsService; import tn.mnlr.vripper.services.VGAuthService; import javax.xml.parsers.SAXParserFactory; @@ -20,20 +21,20 @@ import java.util.concurrent.atomic.AtomicReference; @Slf4j public class ApiPostParser { - private static final String VR_API = "https://vipergirls.to/vr.php"; - private static final SAXParserFactory factory = SAXParserFactory.newInstance(); private final String threadId; private final String postId; private final ConnectionService cm; private final VGAuthService VGAuthService; + private final SettingsService settingsService; public ApiPostParser(String threadId, String postId) { this.threadId = threadId; this.postId = postId; - this.cm = SpringContext.getBean(ConnectionService.class); - this.VGAuthService = SpringContext.getBean(VGAuthService.class); + cm = SpringContext.getBean(ConnectionService.class); + VGAuthService = SpringContext.getBean(VGAuthService.class); + settingsService = SpringContext.getBean(SettingsService.class); } public ApiPost parse() throws PostParseException { @@ -41,7 +42,7 @@ public class ApiPostParser { log.debug(String.format("Parsing post %s", postId)); HttpGet httpGet; try { - URIBuilder uriBuilder = new URIBuilder(VR_API); + URIBuilder uriBuilder = new URIBuilder(settingsService.getSettings().getVProxy() + "/vr.php"); uriBuilder.setParameter("p", postId); httpGet = cm.buildHttpGet(uriBuilder.build(), null); } catch (URISyntaxException e) { diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiThreadHandler.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiThreadHandler.java index ce793a7..b5f405e 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiThreadHandler.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiThreadHandler.java @@ -6,6 +6,7 @@ import org.xml.sax.helpers.DefaultHandler; import tn.mnlr.vripper.SpringContext; import tn.mnlr.vripper.host.Host; import tn.mnlr.vripper.jpa.domain.Queued; +import tn.mnlr.vripper.services.SettingsService; import java.util.*; import java.util.concurrent.atomic.AtomicInteger; @@ -15,6 +16,7 @@ public class ApiThreadHandler extends DefaultHandler { private final Queued queued; private final Collection supportedHosts; + private final SettingsService settingsService; private final Map hostMap = new HashMap<>(); @Getter private final List posts = new ArrayList<>(); @@ -28,7 +30,8 @@ public class ApiThreadHandler extends DefaultHandler { public ApiThreadHandler(Queued queued) { this.queued = queued; - this.supportedHosts = SpringContext.getBeansOfType(Host.class).values(); + supportedHosts = SpringContext.getBeansOfType(Host.class).values(); + settingsService = SpringContext.getBean(SettingsService.class); } @Override @@ -70,7 +73,7 @@ public class ApiThreadHandler extends DefaultHandler { postCounter, postTitle, imageCount, - String.format("https://vipergirls.to/threads/?p=%s&viewfull=1#post%s", postId, postId), + String.format("%s/threads/?p=%s&viewfull=1#post%s", settingsService.getSettings().getVProxy(), postId, postId), previews, hostMap.entrySet().stream().filter(v -> v.getValue().get() > 0).map(e -> e.getKey().getHost() + " (" + e.getValue().get() + ")").collect(Collectors.joining(", ")) )); diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiThreadParser.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiThreadParser.java index 5c41d81..b96ace0 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiThreadParser.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/ApiThreadParser.java @@ -12,6 +12,7 @@ import tn.mnlr.vripper.exception.DownloadException; import tn.mnlr.vripper.exception.PostParseException; import tn.mnlr.vripper.jpa.domain.Queued; import tn.mnlr.vripper.services.ConnectionService; +import tn.mnlr.vripper.services.SettingsService; import tn.mnlr.vripper.services.VGAuthService; import javax.xml.parsers.SAXParserFactory; @@ -23,17 +24,17 @@ import java.util.concurrent.atomic.AtomicReference; @Slf4j public class ApiThreadParser { - private static final String VR_API = "https://vipergirls.to/vr.php"; - private static final SAXParserFactory factory = SAXParserFactory.newInstance(); private final Queued queued; private final ConnectionService cm; private final VGAuthService VGAuthService; + private final SettingsService settingsService; public ApiThreadParser(Queued queued) { this.queued = queued; - this.cm = SpringContext.getBean(ConnectionService.class); - this.VGAuthService = SpringContext.getBean(VGAuthService.class); + cm = SpringContext.getBean(ConnectionService.class); + VGAuthService = SpringContext.getBean(VGAuthService.class); + settingsService = SpringContext.getBean(SettingsService.class); } public List parse() throws PostParseException { @@ -41,7 +42,7 @@ public class ApiThreadParser { log.debug(String.format("Parsing thread %s", queued)); HttpGet httpGet; try { - URIBuilder uriBuilder = new URIBuilder(VR_API); + URIBuilder uriBuilder = new URIBuilder(settingsService.getSettings().getVProxy() + "/vr.php"); uriBuilder.setParameter("t", queued.getThreadId()); httpGet = cm.buildHttpGet(uriBuilder.build(), null); } catch (URISyntaxException e) { diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/MetadataRunnable.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/MetadataRunnable.java deleted file mode 100644 index 8fec657..0000000 --- a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/MetadataRunnable.java +++ /dev/null @@ -1,45 +0,0 @@ -package tn.mnlr.vripper.services.domain; - -import lombok.Getter; -import lombok.NonNull; -import lombok.extern.slf4j.Slf4j; -import net.jodah.failsafe.FailsafeException; -import org.apache.http.impl.execchain.RequestAbortedException; -import tn.mnlr.vripper.SpringContext; -import tn.mnlr.vripper.jpa.domain.Metadata; -import tn.mnlr.vripper.jpa.domain.Post; -import tn.mnlr.vripper.services.DataService; -import tn.mnlr.vripper.services.MetadataService; - -@Slf4j -public class MetadataRunnable implements Runnable { - - @Getter - private final Post post; - - private final MetadataService metadataService; - private final DataService dataService; - - public MetadataRunnable(@NonNull Post post) { - this.post = post; - this.metadataService = SpringContext.getBean(MetadataService.class); - this.dataService = SpringContext.getBean(DataService.class); - } - - @Override - public void run() { - try { - if (Thread.interrupted()) { - log.debug(String.format("Metadata fetching for postId=%s, threadId=%s interrupted", post.getPostId(), post.getThreadId())); - return; - } - Metadata metadata = metadataService.get(post); - dataService.setMetadata(post, metadata); - } catch (Exception e) { - if (e.getCause() instanceof InterruptedException || (e.getCause() instanceof FailsafeException && (e.getCause().getCause() instanceof InterruptedException || e.getCause().getCause() instanceof RequestAbortedException))) { - return; - } - log.error(String.format("Failed to get metadata for postId %s", post.getPostId()), e); - } - } -} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddPostRunnable.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddPostRunnable.java new file mode 100644 index 0000000..7119e02 --- /dev/null +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddPostRunnable.java @@ -0,0 +1,139 @@ +package tn.mnlr.vripper.services.domain.tasks; + +import lombok.extern.slf4j.Slf4j; +import tn.mnlr.vripper.SpringContext; +import tn.mnlr.vripper.Utils; +import tn.mnlr.vripper.download.PendingQueue; +import tn.mnlr.vripper.exception.PostParseException; +import tn.mnlr.vripper.jpa.domain.Event; +import tn.mnlr.vripper.jpa.domain.Image; +import tn.mnlr.vripper.jpa.domain.Post; +import tn.mnlr.vripper.jpa.domain.enums.Status; +import tn.mnlr.vripper.jpa.repositories.IEventRepository; +import tn.mnlr.vripper.services.DataService; +import tn.mnlr.vripper.services.MetadataService; +import tn.mnlr.vripper.services.SettingsService; +import tn.mnlr.vripper.services.VGAuthService; +import tn.mnlr.vripper.services.domain.ApiPost; +import tn.mnlr.vripper.services.domain.ApiPostParser; + +import java.time.LocalDateTime; +import java.util.Set; + +import static tn.mnlr.vripper.jpa.domain.Event.Status.ERROR; +import static tn.mnlr.vripper.jpa.domain.Event.Status.PROCESSING; + +@Slf4j +public class AddPostRunnable implements Runnable { + + private final String postId; + private final String threadId; + private final DataService dataService; + private final MetadataService metadataService; + private final SettingsService settingsService; + private final PendingQueue pendingQueue; + private final VGAuthService VGAuthService; + private final Event event; + private final IEventRepository eventRepository; + private final String link; + + public AddPostRunnable(String postId, String threadId) { + this.postId = postId; + this.threadId = threadId; + this.dataService = SpringContext.getBean(DataService.class); + this.metadataService = SpringContext.getBean(MetadataService.class); + this.settingsService = SpringContext.getBean(SettingsService.class); + this.pendingQueue = SpringContext.getBean(PendingQueue.class); + this.VGAuthService = SpringContext.getBean(VGAuthService.class); + this.eventRepository = SpringContext.getBean(IEventRepository.class); + link = settingsService.getSettings().getVProxy() + String.format("/%s?%s", threadId, (postId != null ? "p=" + postId : "")); + event = new Event(Event.Type.POST, Event.Status.PENDING, LocalDateTime.now(), String.format("Processing %s", link)); + eventRepository.save(event); + } + + @Override + public void run() { + + try { + event.setStatus(PROCESSING); + eventRepository.update(event); + if (dataService.exists(postId)) { + log.warn(String.format("skipping %s, already loaded", postId)); + event.setMessage(String.format("Gallery %s is already loaded", link)); + event.setStatus(ERROR); + eventRepository.update(event); + return; + } + + ApiPostParser apiPostParser = new ApiPostParser(threadId, postId); + + ApiPost apiPost; + try { + apiPost = apiPostParser.parse(); + } catch (PostParseException e) { + String error = String.format("parsing failed for gallery %s", link); + log.error(error, e); + event.setMessage(error + "\n" + Utils.throwableToString(e)); + event.setStatus(ERROR); + eventRepository.update(event); + return; + } + if (apiPost.getPost().isEmpty()) { + String error = String.format("Gallery %s contains no galleries", link); + log.error(error); + event.setMessage(error); + event.setStatus(ERROR); + eventRepository.update(event); + return; + } + if (apiPost.getImages().isEmpty()) { + String error = String.format("Gallery %s contains no images to download", link); + log.error(error); + event.setMessage(error); + event.setStatus(ERROR); + eventRepository.update(event); + return; + } + + Post post = apiPost.getPost().get(); + Set images = apiPost.getImages(); + + dataService.newPost(post, images); + + metadataService.startFetchingMetadata(post); + + if (settingsService.getSettings().getAutoStart()) { + log.debug("Auto start downloads option is enabled"); + post.setStatus(Status.PENDING); + try { + pendingQueue.enqueue(post, images); + } catch (InterruptedException e) { + log.warn("Interruption was caught"); + Thread.currentThread().interrupt(); + return; + } + log.debug(String.format("Done enqueuing jobs for %s", post.getUrl())); + } else { + post.setStatus(Status.STOPPED); + log.debug("Auto start downloads option is disabled"); + } + if (!settingsService.getSettings().getLeaveThanksOnStart()) { + VGAuthService.leaveThanks(post); + } + dataService.updatePostStatus(post.getStatus(), post.getId()); + event.setMessage(String.format("Gallery %s is successfully added to download queue", link)); + event.setStatus(Event.Status.DONE); + eventRepository.update(event); + } catch (Exception e) { + try { + String error = String.format("Error when adding gallery %s", link); + log.error(error, e); + event.setMessage(error + "\n" + Utils.throwableToString(e)); + event.setStatus(ERROR); + eventRepository.update(event); + } catch (Exception exp) { + log.error(exp.getMessage(), exp); + } + } + } +} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddQueuedRunnable.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddQueuedRunnable.java new file mode 100644 index 0000000..918af6d --- /dev/null +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/AddQueuedRunnable.java @@ -0,0 +1,83 @@ +package tn.mnlr.vripper.services.domain.tasks; + +import lombok.extern.slf4j.Slf4j; +import tn.mnlr.vripper.SpringContext; +import tn.mnlr.vripper.Utils; +import tn.mnlr.vripper.jpa.domain.Event; +import tn.mnlr.vripper.jpa.domain.Queued; +import tn.mnlr.vripper.jpa.repositories.IEventRepository; +import tn.mnlr.vripper.services.DataService; +import tn.mnlr.vripper.services.PostService; +import tn.mnlr.vripper.services.ThreadPoolService; +import tn.mnlr.vripper.services.domain.MultiPostItem; + +import java.time.LocalDateTime; +import java.util.List; + +import static tn.mnlr.vripper.jpa.domain.Event.Status.ERROR; +import static tn.mnlr.vripper.jpa.domain.Event.Status.PROCESSING; + +@Slf4j +public class AddQueuedRunnable implements Runnable { + + private final Queued queued; + private final ThreadPoolService threadPoolService; + private final DataService dataService; + private final IEventRepository eventRepository; + private final PostService postService; + private final Event event; + + public AddQueuedRunnable(Queued queued) { + this.queued = queued; + threadPoolService = SpringContext.getBean(ThreadPoolService.class); + dataService = SpringContext.getBean(DataService.class); + eventRepository = SpringContext.getBean(IEventRepository.class); + postService = SpringContext.getBean(PostService.class); + event = new Event(Event.Type.QUEUED, Event.Status.PENDING, LocalDateTime.now(), String.format("Processing multi-post link %s", queued.getLink())); + eventRepository.save(event); + } + + @Override + public void run() { + try { + event.setStatus(PROCESSING); + eventRepository.update(event); + List multiPostItems = postService.get(queued); + if (multiPostItems == null) { + String message = String.format("Fetching multi-post link %s failed", queued.getLink()); + event.setStatus(ERROR); + event.setMessage(message); + eventRepository.update(event); + return; + } + queued.setTotal(multiPostItems.size()); + queued.done(); + if (multiPostItems.size() == 1) { + threadPoolService.getGeneralExecutor().submit(new AddPostRunnable(multiPostItems.get(0).getPostId(), multiPostItems.get(0).getThreadId())); + event.setStatus(Event.Status.DONE); + event.setMessage(String.format("Link %s is added to download queue", queued.getLink())); + } else { + if (dataService.findQueuedByThreadId(queued.getThreadId()).isEmpty()) { + dataService.newQueueLink(queued); + event.setStatus(Event.Status.DONE); + event.setMessage(String.format("Link %s is added to multi-post links", queued.getLink())); + } else { + log.info(String.format("Link %s is already loaded", queued.getLink())); + event.setStatus(Event.Status.ERROR); + event.setMessage(String.format("%s has already been added to multi-post links", queued.getLink())); + } + } + eventRepository.update(event); + } catch (Exception e) { + try { + String error = String.format("Error when adding multi-post link %s", queued.getLink()); + log.error(error, e); + event.setMessage(error + "\n" + Utils.throwableToString(e)); + event.setStatus(ERROR); + eventRepository.update(event); + } catch (Exception exp) { + log.error(exp.getMessage(), exp); + } + } + } +} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LeaveThanksRunnable.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LeaveThanksRunnable.java new file mode 100644 index 0000000..124a474 --- /dev/null +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LeaveThanksRunnable.java @@ -0,0 +1,123 @@ +package tn.mnlr.vripper.services.domain.tasks; + +import lombok.extern.slf4j.Slf4j; +import org.apache.http.NameValuePair; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; +import tn.mnlr.vripper.SpringContext; +import tn.mnlr.vripper.Utils; +import tn.mnlr.vripper.jpa.domain.Event; +import tn.mnlr.vripper.jpa.domain.Post; +import tn.mnlr.vripper.jpa.repositories.IEventRepository; +import tn.mnlr.vripper.services.ConnectionService; +import tn.mnlr.vripper.services.DataService; +import tn.mnlr.vripper.services.SettingsService; + +import java.io.UnsupportedEncodingException; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; + +@Slf4j +public class LeaveThanksRunnable implements Runnable { + + private final ConnectionService cm; + private final DataService dataService; + private final HttpClientContext context; + private final SettingsService settingsService; + private final IEventRepository eventRepository; + private final boolean authenticated; + private final Post post; + private final Event event; + + public LeaveThanksRunnable(Post post, boolean authenticated, HttpClientContext context) { + this.post = post; + this.authenticated = authenticated; + this.context = context; + cm = SpringContext.getBean(ConnectionService.class); + dataService = SpringContext.getBean(DataService.class); + eventRepository = SpringContext.getBean(IEventRepository.class); + settingsService = SpringContext.getBean(SettingsService.class); + event = new Event(Event.Type.THANKS, Event.Status.PENDING, LocalDateTime.now(), String.format("Leaving thanks for %s", post.getUrl())); + eventRepository.save(event); + } + + @Override + public void run() { + try { + event.setStatus(Event.Status.PROCESSING); + eventRepository.update(event); + if (!settingsService.getSettings().getVLogin()) { + event.setMessage(String.format("Will not leave a thanks for %s\nAuthentication with ViperGirls option is disabled", post.getUrl())); + event.setStatus(Event.Status.DONE); + eventRepository.update(event); + return; + } + if (!settingsService.getSettings().getVThanks()) { + event.setMessage(String.format("Will not leave a thanks for %s\nLeave thanks option is disabled", post.getUrl())); + event.setStatus(Event.Status.DONE); + eventRepository.update(event); + return; + } + if (!authenticated) { + event.setMessage(String.format("Will not leave a thanks for %s\nYou are not authenticated", post.getUrl())); + event.setStatus(Event.Status.ERROR); + eventRepository.update(event); + return; + } + if (post.isThanked()) { + event.setMessage(String.format("Will not leave a thanks for %s\nAlready left a thanks", post.getUrl())); + event.setStatus(Event.Status.DONE); + eventRepository.update(event); + return; + } + + HttpPost postThanks = cm.buildHttpPost(settingsService.getSettings().getVProxy() + "/post_thanks.php", null); + List params = new ArrayList<>(); + params.add(new BasicNameValuePair("do", "post_thanks_add")); + params.add(new BasicNameValuePair("using_ajax", "1")); + params.add(new BasicNameValuePair("p", post.getPostId())); + params.add(new BasicNameValuePair("securitytoken", post.getSecurityToken())); + try { + postThanks.setEntity(new UrlEncodedFormEntity(params)); + } catch (UnsupportedEncodingException e) { + String error = String.format("Request error for %s", post.getUrl()); + log.error(error, e); + event.setMessage(error + "\n" + Utils.throwableToString(e)); + event.setStatus(Event.Status.ERROR); + eventRepository.update(event); + return; + } + + postThanks.addHeader("Referer", settingsService.getSettings().getVProxy()); + postThanks.addHeader("Host", settingsService.getSettings().getVProxy().replace("https://", "").replace("http://", "")); + + CloseableHttpClient client = cm.getClient().build(); + + try (CloseableHttpResponse response = client.execute(postThanks, context)) { + if (response.getStatusLine().getStatusCode() / 100 == 2) { + post.setThanked(true); + dataService.updatePostThanked(post.isThanked(), post.getId()); + } + EntityUtils.consumeQuietly(response.getEntity()); + } + event.setStatus(Event.Status.DONE); + eventRepository.update(event); + } catch (Exception e) { + try { + String error = String.format("Failed to leave a thanks for %s", post.getUrl()); + log.error(error, e); + event.setMessage(error + "\n" + Utils.throwableToString(e)); + event.setStatus(Event.Status.ERROR); + eventRepository.update(event); + } catch (Exception exp) { + log.error(exp.getMessage(), exp); + } + } + } +} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LinkScanRunnable.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LinkScanRunnable.java new file mode 100644 index 0000000..58d7779 --- /dev/null +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/LinkScanRunnable.java @@ -0,0 +1,99 @@ +package tn.mnlr.vripper.services.domain.tasks; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.lang.NonNull; +import tn.mnlr.vripper.SpringContext; +import tn.mnlr.vripper.Utils; +import tn.mnlr.vripper.jpa.domain.Event; +import tn.mnlr.vripper.jpa.domain.Queued; +import tn.mnlr.vripper.jpa.repositories.impl.EventRepository; +import tn.mnlr.vripper.services.PostService; +import tn.mnlr.vripper.services.SettingsService; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static tn.mnlr.vripper.jpa.domain.Event.Status.ERROR; + +@Slf4j +public class LinkScanRunnable implements Runnable { + + private final static Object LOCK = new Object(); + private final List urlList; + private final SettingsService settingsService; + private final PostService postService; + private final EventRepository eventRepository; + private final Event event; + + + public LinkScanRunnable(@NonNull List urlList) { + this.urlList = urlList; + settingsService = SpringContext.getBean(SettingsService.class); + postService = SpringContext.getBean(PostService.class); + eventRepository = SpringContext.getBean(EventRepository.class); + event = new Event(Event.Type.SCAN, Event.Status.PENDING, LocalDateTime.now(), "Links scan:\n\t" + String.join("\n\t", urlList)); + eventRepository.save(event); + } + + @Override + public void run() { + synchronized (LOCK) { + try { + event.setStatus(Event.Status.PROCESSING); + eventRepository.update(event); + ArrayList queuedList = new ArrayList<>(); + List unsupported = new ArrayList<>(); + List unrecognized = new ArrayList<>(); + for (String url : urlList) { + log.debug(String.format("Starting to process thread: %s", url)); + if (!url.startsWith(settingsService.getSettings().getVProxy())) { + log.error(String.format("Unsupported link %s", url)); + unsupported.add(url); + continue; + } + + String threadId, postId; + Matcher m = Pattern.compile(Pattern.quote(settingsService.getSettings().getVProxy()) + "/threads/(\\d+)((.*p=)(\\d+))?").matcher(url); + if (m.find()) { + threadId = m.group(1); + postId = m.group(4); + } else { + log.error(String.format("Cannot retrieve thread id from URL %s", url)); + unrecognized.add(url); + continue; + } + queuedList.add(new Queued(url, threadId, postId)); + } + StringBuilder errorMessage = new StringBuilder(); + if (!unsupported.isEmpty()) { + errorMessage.append("Unsupported links:\n\t").append(String.join("\n\t", unsupported)).append("\n\n"); + } + if (!unrecognized.isEmpty()) { + errorMessage.append("Unrecognized links:\n\t").append(String.join("\n\t", unrecognized)).append("\n\n"); + } + + postService.processMultiPost(queuedList); + if (!unsupported.isEmpty() || !unrecognized.isEmpty()) { + event.setStatus(Event.Status.ERROR); + event.setMessage("Some links failed to be scanned: \n" + errorMessage.toString()); + } else { + event.setStatus(Event.Status.DONE); + } + eventRepository.update(event); + } catch (Exception e) { + try { + String error = "Error when scanning links"; + log.error(error, e); + event.setMessage(error + "\n" + Utils.throwableToString(e)); + event.setStatus(ERROR); + eventRepository.update(event); + } catch (Exception exp) { + log.error(exp.getMessage(), exp); + } + } + } + } +} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/MetadataRunnable.java b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/MetadataRunnable.java new file mode 100644 index 0000000..d442005 --- /dev/null +++ b/vripper-server/src/main/java/tn/mnlr/vripper/services/domain/tasks/MetadataRunnable.java @@ -0,0 +1,82 @@ +package tn.mnlr.vripper.services.domain.tasks; + +import lombok.Getter; +import lombok.NonNull; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import tn.mnlr.vripper.SpringContext; +import tn.mnlr.vripper.Utils; +import tn.mnlr.vripper.jpa.domain.Event; +import tn.mnlr.vripper.jpa.domain.Metadata; +import tn.mnlr.vripper.jpa.domain.Post; +import tn.mnlr.vripper.jpa.repositories.IEventRepository; +import tn.mnlr.vripper.services.DataService; +import tn.mnlr.vripper.services.MetadataService; + +import java.time.LocalDateTime; + +import static tn.mnlr.vripper.jpa.domain.Event.Status.ERROR; + +@Slf4j +public class MetadataRunnable implements Runnable { + + @Getter + private final Post post; + + private final MetadataService metadataService; + private final DataService dataService; + private final IEventRepository eventRepository; + + private final Event event; + + @Setter + private volatile boolean interrupted = false; + + public MetadataRunnable(@NonNull Post post) { + this.post = post; + metadataService = SpringContext.getBean(MetadataService.class); + dataService = SpringContext.getBean(DataService.class); + eventRepository = SpringContext.getBean(IEventRepository.class); + event = new Event(Event.Type.METADATA, Event.Status.PENDING, LocalDateTime.now(), "Fetching metadata for " + post.getUrl()); + eventRepository.save(event); + } + + @Override + public void run() { + + try { + event.setStatus(Event.Status.PROCESSING); + eventRepository.update(event); + if (interrupted) { + String message = String.format("Fetching metadata for %s interrupted", post.getUrl()); + event.setStatus(Event.Status.DONE); + event.setMessage(message); + eventRepository.update(event); + return; + } + + Metadata metadata = metadataService.get(post); + if (metadata == null) { + String message = String.format("Fetching metadata for %s failed", post.getUrl()); + event.setStatus(ERROR); + event.setMessage(message); + eventRepository.update(event); + return; + } + dataService.setMetadata(post, metadata); + + event.setStatus(Event.Status.DONE); + eventRepository.update(event); + } catch (Exception e) { + try { + String message = String.format("Failed to fetch metadata for %s", post.getUrl()); + log.error(message, e); + event.setMessage(message + "\n" + Utils.throwableToString(e)); + event.setStatus(ERROR); + eventRepository.update(event); + } catch (Exception exp) { + log.error(exp.getMessage(), exp); + } + } + } +} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/EventLogRestEndpoint.java b/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/EventLogRestEndpoint.java new file mode 100644 index 0000000..7239f1c --- /dev/null +++ b/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/EventLogRestEndpoint.java @@ -0,0 +1,27 @@ +package tn.mnlr.vripper.web.restendpoints; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; +import tn.mnlr.vripper.jpa.repositories.IEventRepository; + +@Slf4j +@RestController +@CrossOrigin(value = "*") +public class EventLogRestEndpoint { + + private final IEventRepository eventRepository; + + public EventLogRestEndpoint(IEventRepository eventRepository) { + this.eventRepository = eventRepository; + } + + @GetMapping("/events/clear") + @ResponseStatus(value = HttpStatus.OK) + public void clear() { + eventRepository.deleteAll(); + } +} diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java b/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java index 1b666ec..785cb9d 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/PostRestEndpoint.java @@ -6,15 +6,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.*; import tn.mnlr.vripper.download.DownloadService; -import tn.mnlr.vripper.exception.PostParseException; import tn.mnlr.vripper.jpa.domain.Metadata; import tn.mnlr.vripper.jpa.domain.Post; import tn.mnlr.vripper.jpa.domain.Queued; -import tn.mnlr.vripper.services.DataService; -import tn.mnlr.vripper.services.PathService; -import tn.mnlr.vripper.services.PostService; -import tn.mnlr.vripper.services.ThreadPoolService; +import tn.mnlr.vripper.services.*; import tn.mnlr.vripper.services.domain.MultiPostItem; +import tn.mnlr.vripper.services.domain.tasks.AddPostRunnable; +import tn.mnlr.vripper.services.domain.tasks.LinkScanRunnable; import tn.mnlr.vripper.web.restendpoints.domain.*; import tn.mnlr.vripper.web.restendpoints.exceptions.BadRequestException; import tn.mnlr.vripper.web.restendpoints.exceptions.NotFoundException; @@ -24,9 +22,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Optional; -import java.util.concurrent.ExecutionException; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; @Slf4j @@ -34,21 +29,22 @@ import java.util.stream.Collectors; @CrossOrigin(value = "*") public class PostRestEndpoint { - private static final Pattern VG_URL_PATTERN = Pattern.compile("https://vipergirls\\.to/threads/(\\d+)((.*p=)(\\d+))?"); private static final Object LOCK = new Object(); private final DataService dataService; private final PathService pathService; private final DownloadService downloadService; private final PostService postService; private final ThreadPoolService threadPoolService; + private final SettingsService settingsService; @Autowired - public PostRestEndpoint(DataService dataService, PathService pathService, DownloadService downloadService, PostService postService, ThreadPoolService threadPoolService) { + public PostRestEndpoint(DataService dataService, PathService pathService, DownloadService downloadService, PostService postService, ThreadPoolService threadPoolService, SettingsService settingsService) { this.dataService = dataService; this.pathService = pathService; this.downloadService = downloadService; this.postService = postService; this.threadPoolService = threadPoolService; + this.settingsService = settingsService; } @PostMapping("/post") @@ -60,30 +56,7 @@ public class PostRestEndpoint { throw new BadRequestException("Cannot process empty requests"); } List urlList = Arrays.stream(_url.getUrl().split("\\r?\\n")).map(String::trim).filter(e -> !e.isEmpty()).collect(Collectors.toList()); - ArrayList queuedList = new ArrayList<>(); - for (String url : urlList) { - log.debug(String.format("Starting to process thread: %s", url)); - if (!url.startsWith("https://vipergirls.to")) { - log.error(String.format("Unsupported link %s", url)); - continue; - } - - String threadId, postId; - Matcher m = VG_URL_PATTERN.matcher(url); - if (m.find()) { - threadId = m.group(1); - postId = m.group(4); - } else { - throw new BadRequestException(String.format("Cannot retrieve thread id from URL %s", url)); - } - queuedList.add(new Queued(url, threadId, postId)); - } - try { - postService.processMultiPost(queuedList); - } catch (Exception e) { - log.error("Failed to parse links", e); - throw new ServerErrorException(e.getMessage()); - } + threadPoolService.getGeneralExecutor().submit(new LinkScanRunnable(urlList)); } } @@ -100,14 +73,7 @@ public class PostRestEndpoint { public void addPost(@RequestBody List posts) { synchronized (LOCK) { for (PostToAdd post : posts) { - threadPoolService.getGeneralExecutor().submit(() -> { - try { - postService.addPost(post.getPostId(), post.getThreadId()); - } catch (PostParseException e) { - log.error(String.format("Failed to add post %s", post.getPostId()), e); - throw new ServerErrorException(String.format("Failed to add post %s", post.getPostId())); - } - }); + threadPoolService.getGeneralExecutor().submit(new AddPostRunnable(post.getPostId(), post.getThreadId())); } } } @@ -239,11 +205,12 @@ public class PostRestEndpoint { public List grab(@PathVariable("threadId") @NonNull String threadId) { synchronized (LOCK) { Queued queued = dataService.findQueuedByThreadId(threadId).orElseThrow(() -> new NotFoundException(String.format("Unable to find links for threadId = %s", threadId))); - try { - return postService.getCache().get(queued); - } catch (ExecutionException e) { - log.error(String.format("Failed to get links for threadId = %s", threadId), e); + List multiPostItems = postService.get(queued); + if (multiPostItems == null) { + log.error(String.format("Failed to get links for threadId = %s", threadId)); throw new ServerErrorException(String.format("Failed to get links for threadId = %s", threadId)); + } else { + return multiPostItems; } } } @@ -256,4 +223,12 @@ public class PostRestEndpoint { return threadId; } } + + @GetMapping("/grab/clear") + @ResponseStatus(value = HttpStatus.OK) + public void grabClear() { + synchronized (LOCK) { + dataService.clearQueueLinks(); + } + } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java b/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java index 67d0a9a..ea07153 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/web/restendpoints/SettingsRestEndpoint.java @@ -8,6 +8,8 @@ import tn.mnlr.vripper.exception.ValidationException; import tn.mnlr.vripper.services.SettingsService; import tn.mnlr.vripper.web.restendpoints.exceptions.BadRequestException; +import java.util.List; + @RestController @Slf4j @CrossOrigin(value = "*") @@ -35,7 +37,7 @@ public class SettingsRestEndpoint { @PostMapping("/settings") @ResponseStatus(value = HttpStatus.OK) - public SettingsService.Settings postSettings(@RequestBody SettingsService.Settings settings) throws Exception { + public SettingsService.Settings postSettings(@RequestBody SettingsService.Settings settings) { try { this.settingsService.check(settings); @@ -54,4 +56,10 @@ public class SettingsRestEndpoint { return settingsService.getSettings(); } + + @GetMapping("/settings/proxies") + @ResponseStatus(value = HttpStatus.OK) + public List mirrors() { + return settingsService.getProxies(); + } } diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/DataBroadcast.java b/vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/DataBroadcast.java index 1f0b257..27f8f39 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/DataBroadcast.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/DataBroadcast.java @@ -38,14 +38,16 @@ public class DataBroadcast { private final MetadataUpdateEventListener metadataUpdateEventListener; private final ImageUpdateEventListener imageUpdateEventListener; private final QueuedUpdateEventListener queuedUpdateEventListener; + private final EventUpdateEventListener eventUpdateEventListener; private final QueuedRemoveEventListener queuedRemoveEventListener; private final PostRemoveEventListener postRemoveEventListener; + private final EventRemoveEventListener eventRemoveEventListener; private final List disposables = new ArrayList<>(); @Autowired - public DataBroadcast(SimpMessagingTemplate template, VGAuthService VGAuthService, GlobalStateService globalStateService, DownloadSpeedService downloadSpeedService, DataService dataService, PostUpdateEventListener postUpdateEventListener, MetadataUpdateEventListener metadataUpdateEventListener, ImageUpdateEventListener imageUpdateEventListener, QueuedUpdateEventListener queuedUpdateEventListener, QueuedRemoveEventListener queuedRemoveEventListener, PostRemoveEventListener postRemoveEventListener) { + public DataBroadcast(SimpMessagingTemplate template, VGAuthService VGAuthService, GlobalStateService globalStateService, DownloadSpeedService downloadSpeedService, DataService dataService, PostUpdateEventListener postUpdateEventListener, MetadataUpdateEventListener metadataUpdateEventListener, ImageUpdateEventListener imageUpdateEventListener, QueuedUpdateEventListener queuedUpdateEventListener, EventUpdateEventListener eventUpdateEventListener, QueuedRemoveEventListener queuedRemoveEventListener, PostRemoveEventListener postRemoveEventListener, EventRemoveEventListener eventRemoveEventListener) { this.template = template; this.VGAuthService = VGAuthService; this.globalStateService = globalStateService; @@ -55,8 +57,10 @@ public class DataBroadcast { this.metadataUpdateEventListener = metadataUpdateEventListener; this.imageUpdateEventListener = imageUpdateEventListener; this.queuedUpdateEventListener = queuedUpdateEventListener; + this.eventUpdateEventListener = eventUpdateEventListener; this.queuedRemoveEventListener = queuedRemoveEventListener; this.postRemoveEventListener = postRemoveEventListener; + this.eventRemoveEventListener = eventRemoveEventListener; } @PostConstruct @@ -86,6 +90,13 @@ public class DataBroadcast { .filter(e -> !e.isEmpty()) .subscribe(ids -> template.convertAndSend("/topic/posts", ids.stream().map(dataService::findById).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList())), e -> log.error("Failed to send data to client", e))); + disposables.add(eventUpdateEventListener.getDataFlux() + .map(EventUpdateEvent::getId) + .buffer(Duration.of(500, ChronoUnit.MILLIS)) + .map(HashSet::new) + .filter(e -> !e.isEmpty()) + .subscribe(ids -> template.convertAndSend("/topic/events", ids.stream().map(dataService::findEventById).filter(Optional::isPresent).map(Optional::get).collect(Collectors.toList())), e -> log.error("Failed to send data to client", e))); + disposables.add(imageUpdateEventListener.getDataFlux() .map(ImageUpdateEvent::getId) .buffer(Duration.of(500, ChronoUnit.MILLIS)) @@ -124,6 +135,14 @@ public class DataBroadcast { .filter(e -> !e.isEmpty()) .subscribe(postIds -> template.convertAndSend("/topic/posts/deleted", postIds), e -> log.error("Failed to send data to client", e)) ); + + disposables.add(eventRemoveEventListener.getDataFlux() + .map(EventRemoveEvent::getId) + .buffer(Duration.of(500, ChronoUnit.MILLIS)) + .map(HashSet::new) + .filter(e -> !e.isEmpty()) + .subscribe(postIds -> template.convertAndSend("/topic/events/deleted", postIds), e -> log.error("Failed to send data to client", e)) + ); } @PreDestroy diff --git a/vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/DataController.java b/vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/DataController.java index 586027e..185c9a5 100644 --- a/vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/DataController.java +++ b/vripper-server/src/main/java/tn/mnlr/vripper/web/wsendpoints/DataController.java @@ -5,6 +5,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.messaging.handler.annotation.DestinationVariable; import org.springframework.messaging.simp.annotation.SubscribeMapping; import org.springframework.stereotype.Controller; +import tn.mnlr.vripper.jpa.domain.Event; import tn.mnlr.vripper.jpa.domain.Image; import tn.mnlr.vripper.jpa.domain.Post; import tn.mnlr.vripper.jpa.domain.Queued; @@ -17,8 +18,6 @@ import tn.mnlr.vripper.services.domain.GlobalState; import java.util.Collection; import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; @Controller public class DataController { @@ -53,7 +52,7 @@ public class DataController { @SubscribeMapping("/posts") public Collection posts() { - return StreamSupport.stream(dataService.findAllPosts().spliterator(), false).collect(Collectors.toList()); + return dataService.findAllPosts(); } @SubscribeMapping("/images/{postId}") @@ -63,7 +62,12 @@ public class DataController { @SubscribeMapping("/queued") public Collection queued() { - return StreamSupport.stream(dataService.findAllQueued().spliterator(), false).collect(Collectors.toList()); + return dataService.findAllQueued(); + } + + @SubscribeMapping("/events") + public Collection events() { + return dataService.findAllEvents(); } @Getter diff --git a/vripper-server/src/main/resources/application.properties b/vripper-server/src/main/resources/application.properties index d606b4d..45ed67d 100644 --- a/vripper-server/src/main/resources/application.properties +++ b/vripper-server/src/main/resources/application.properties @@ -8,7 +8,7 @@ management.endpoints.web.exposure.include=shutdown management.endpoint.shutdown.enabled=true server.error.include-message=always spring.liquibase.change-log=classpath:db/changelog/db.changelog-master.xml -spring.datasource.url=jdbc:hsqldb:file:${base.dir}/${base.dir.name}/db/xparty;hsqldb.lock_file=false +spring.datasource.url=jdbc:hsqldb:file:${base.dir}/${base.dir.name}/db/vripper;hsqldb.lock_file=false spring.datasource.username=SA spring.datasource.password=lEtmEIn spring.datasource.driver-class-name=org.hsqldb.jdbc.JDBCDriver diff --git a/vripper-server/src/main/resources/db/changelog/db.changelog-master.xml b/vripper-server/src/main/resources/db/changelog/db.changelog-master.xml index d7c0c96..44925ed 100644 --- a/vripper-server/src/main/resources/db/changelog/db.changelog-master.xml +++ b/vripper-server/src/main/resources/db/changelog/db.changelog-master.xml @@ -2,10 +2,60 @@ - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -31,61 +81,10 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -104,69 +103,62 @@ + + + + + + + + + + + + + + + + + - - + + + + + - - + + + + + + - - - - - - - + - - - - - - - - - - - - - - - - - - - + - - - - - - - diff --git a/vripper-server/src/main/resources/logo.png b/vripper-server/src/main/resources/logo.png deleted file mode 100644 index e737cd4..0000000 Binary files a/vripper-server/src/main/resources/logo.png and /dev/null differ diff --git a/vripper-server/src/main/resources/proxies.json b/vripper-server/src/main/resources/proxies.json new file mode 100644 index 0000000..ea26382 --- /dev/null +++ b/vripper-server/src/main/resources/proxies.json @@ -0,0 +1,8 @@ +[ + "https://planetviper.club", + "https://viperbb.rocks", + "https://viperkats.eu", + "https://viperohilia.art", + "https://viperproxy.org", + "https://vipervault.link" +] diff --git a/vripper-ui/angular.json b/vripper-ui/angular.json index 86908da..348943c 100644 --- a/vripper-ui/angular.json +++ b/vripper-ui/angular.json @@ -26,10 +26,17 @@ "assets": [ "src/favicon.ico", "src/assets", - { "glob": "mdi.svg", "input": "./node_modules/@mdi/angular-material", "output": "./assets" } + { + "glob": "mdi.svg", + "input": "./node_modules/@mdi/angular-material", + "output": "./assets" + } ], "styles": [ "src/styles.scss" + ], + "allowedCommonJsDependencies": [ + "@stomp/rx-stomp" ] }, "configurations": { diff --git a/vripper-ui/src/app/app.module.ts b/vripper-ui/src/app/app.module.ts index 49a1ea2..5d0dea4 100644 --- a/vripper-ui/src/app/app.module.ts +++ b/vripper-ui/src/app/app.module.ts @@ -27,6 +27,8 @@ import {AlternativeTitleComponent} from './posts/alternative-title/alternative-t import {NgxElectronModule} from 'ngx-electron'; import {MatIconRegistry} from '@angular/material/icon'; import {DomSanitizer} from '@angular/platform-browser'; +import {EventLogComponent} from './event-log/event-log.component'; +import {EventLogMessageDialogComponent} from './event-log/message-dialog/event-log-message-dialog.component'; @NgModule({ declarations: [ @@ -45,6 +47,8 @@ import {DomSanitizer} from '@angular/platform-browser'; MultiPostGridComponent, PostContextMenuComponent, AlternativeTitleComponent, + EventLogComponent, + EventLogMessageDialogComponent ], imports: [ BrowserAnimationsModule, diff --git a/vripper-ui/src/app/domain/event.model.ts b/vripper-ui/src/app/domain/event.model.ts new file mode 100644 index 0000000..c5b2d9a --- /dev/null +++ b/vripper-ui/src/app/domain/event.model.ts @@ -0,0 +1,10 @@ +export class EventLog { + constructor( + public id: number, + public type: string, + public status: string, + public time: string, + public message: string + ) { + } +} diff --git a/vripper-ui/src/app/domain/settings.model.ts b/vripper-ui/src/app/domain/settings.model.ts index 5d16e8b..28dfc9c 100644 --- a/vripper-ui/src/app/domain/settings.model.ts +++ b/vripper-ui/src/app/domain/settings.model.ts @@ -12,4 +12,6 @@ export interface Settings { resolveTitle: boolean; connectionTimeout: number; maxAttempts: number; + vProxy: string; + maxEventLog: number; } diff --git a/vripper-ui/src/app/event-log/event-log.component.html b/vripper-ui/src/app/event-log/event-log.component.html new file mode 100644 index 0000000..fdae3ef --- /dev/null +++ b/vripper-ui/src/app/event-log/event-log.component.html @@ -0,0 +1,4 @@ +
+ + +
diff --git a/vripper-ui/src/app/event-log/event-log.component.ts b/vripper-ui/src/app/event-log/event-log.component.ts new file mode 100644 index 0000000..77c0b89 --- /dev/null +++ b/vripper-ui/src/app/event-log/event-log.component.ts @@ -0,0 +1,106 @@ +import {ChangeDetectionStrategy, Component, NgZone, OnDestroy} from '@angular/core'; +import {GridOptions} from 'ag-grid-community'; +import {WsConnectionService} from '../services/ws-connection.service'; +import {EventLogDatasource} from './event-log.datasource'; +import {StatusRendererNative} from '../grid-custom-cells/status-renderer.native'; +import {EventLogService} from '../services/event-log.service'; +import {EventMessageRendererNative} from '../grid-custom-cells/event-message-renderer.native'; + +@Component({ + selector: 'app-event-log', + templateUrl: './event-log.component.html', + changeDetection: ChangeDetectionStrategy.OnPush +}) +export class EventLogComponent implements OnDestroy { + gridOptions: GridOptions; + dataSource: EventLogDatasource; + + constructor( + private wsConnection: WsConnectionService, + private zone: NgZone, + private eventLogService: EventLogService + ) { + this.gridOptions = { + columnDefs: [ + { + headerName: 'Time', + field: 'time', + sort: 'desc', + width: 250, + maxWidth: 250 + }, { + headerName: 'Type', + field: 'type', + valueGetter: (params => { + switch (params.data.type) { + case 'POST': + return 'New gallery'; + case 'QUEUED': + return 'New multi-post link'; + case 'THANKS': + return 'Leave thanks'; + case 'SCAN': + return 'Links scan'; + case 'METADATA': + return 'Fetching metadata'; + case 'METADATA_CACHE_MISS': + return 'Loading post metadata'; + case 'QUEUED_CACHE_MISS': + return 'Loading multi-post links'; + case 'DOWNLOAD': + return 'Download'; + default: + return params.data.type; + } + }), + width: 250, + maxWidth: 250 + }, { + headerName: 'Status', + field: 'status', + cellRenderer: 'nativeStatusCellRenderer', + width: 150, + maxWidth: 150 + }, { + headerName: 'Message', + field: 'message', + flex: 1, + cellRenderer: 'messageCellRenderer', + cellRendererParams: { + eventLogService: this.eventLogService + }, + } + ], + defaultColDef: { + sortable: true, + resizable: true + }, + rowHeight: 26, + headerHeight: 35, + animateRows: true, + rowSelection: 'single', + rowDeselection: true, + rowData: [], + components: { + nativeStatusCellRenderer: StatusRendererNative, + messageCellRenderer: EventMessageRendererNative, + }, + overlayLoadingTemplate: '', + overlayNoRowsTemplate: '', + getRowNodeId: data => data['id'], + onGridReady: () => { + this.eventLogService.setGridApi(this.gridOptions.api); + this.dataSource = new EventLogDatasource(this.wsConnection, this.gridOptions, this.zone); + this.dataSource.connect(); + }, + onRowDataUpdated: () => this.eventLogService.setCount(this.gridOptions.api.getDisplayedRowCount()), + onRowDataChanged: () => this.eventLogService.setCount(this.gridOptions.api.getDisplayedRowCount()), + }; + } + + ngOnDestroy(): void { + if (this.dataSource) { + this.dataSource.disconnect(); + } + } +} diff --git a/vripper-ui/src/app/event-log/event-log.datasource.ts b/vripper-ui/src/app/event-log/event-log.datasource.ts new file mode 100644 index 0000000..1978d07 --- /dev/null +++ b/vripper-ui/src/app/event-log/event-log.datasource.ts @@ -0,0 +1,48 @@ +import {Subscription} from 'rxjs'; +import {WsConnectionService} from '../services/ws-connection.service'; +import {GridOptions, RowNode} from 'ag-grid-community'; +import {NgZone} from '@angular/core'; +import {EventLog} from '../domain/event.model'; + +export class EventLogDatasource { + subscriptions: Subscription[] = []; + + constructor(private ws: WsConnectionService, private gridOptions: GridOptions, private zone: NgZone) { + } + + connect() { + this.subscriptions.push(this.ws.events$.subscribe((e: EventLog[]) => { + this.zone.run(() => { + const toAdd = []; + const toUpdate = []; + e.forEach(v => { + if (this.gridOptions.api.getRowNode(v.id.toString(10)) == null) { + toAdd.push(v); + } else { + toUpdate.push(v); + } + }); + this.gridOptions.api.applyTransaction({update: toUpdate, add: toAdd}); + }); + })); + + this.subscriptions.push(this.ws.eventsRemove$.subscribe((e: number[]) => { + this.zone.run(() => { + const toRemove = []; + e.forEach(v => { + const rowNode: RowNode = this.gridOptions.api.getRowNode(v.toString(10)); + if (rowNode != null) { + toRemove.push(rowNode.data); + } + return; + }); + this.gridOptions.api.applyTransaction({remove: toRemove}); + }); + })); + } + + disconnect() { + console.log('Disconnecting from events datasource'); + this.subscriptions.forEach(e => e.unsubscribe()); + } +} diff --git a/vripper-ui/src/app/event-log/message-dialog/event-log-message-dialog.component.html b/vripper-ui/src/app/event-log/message-dialog/event-log-message-dialog.component.html new file mode 100644 index 0000000..47f57a9 --- /dev/null +++ b/vripper-ui/src/app/event-log/message-dialog/event-log-message-dialog.component.html @@ -0,0 +1,10 @@ +
+

Message

+ +
{{data}}
+
+ + + +
+ diff --git a/vripper-ui/src/app/event-log/message-dialog/event-log-message-dialog.component.ts b/vripper-ui/src/app/event-log/message-dialog/event-log-message-dialog.component.ts new file mode 100644 index 0000000..aead7a8 --- /dev/null +++ b/vripper-ui/src/app/event-log/message-dialog/event-log-message-dialog.component.ts @@ -0,0 +1,19 @@ +import {ChangeDetectionStrategy, Component, Inject, NgZone} from '@angular/core'; +import {MAT_DIALOG_DATA, MatDialogRef} from '@angular/material/dialog'; + +@Component({ + selector: 'app-event-log-message-dialog', + templateUrl: './event-log-message-dialog.component.html', + changeDetection: ChangeDetectionStrategy.OnPush +}) +export class EventLogMessageDialogComponent { + + constructor(public dialogRef: MatDialogRef, + @Inject(MAT_DIALOG_DATA) public data: string, + private ngZone: NgZone) { + } + + close() { + this.ngZone.run(() => this.dialogRef.close()); + } +} diff --git a/vripper-ui/src/app/grid-custom-cells/collector-actions-renderer.native.ts b/vripper-ui/src/app/grid-custom-cells/collector-actions-renderer.native.ts index 4efde58..4f52049 100644 --- a/vripper-ui/src/app/grid-custom-cells/collector-actions-renderer.native.ts +++ b/vripper-ui/src/app/grid-custom-cells/collector-actions-renderer.native.ts @@ -32,14 +32,14 @@ export class CollectorActionsRendererNative implements ICellRendererComp { this.selectButton = document.createElement('button'); this.selectButton.classList.add('mat-icon-button', 'mat-button-base', 'cell-icon'); this.selectButton.setAttribute('style', 'width: auto; height: 24px; display: flex; align-items: center'); - this.selectButton.innerHTML = `Select`; + this.selectButton.innerHTML = `Select`; this.selectSpan.append(this.selectButton); this.deleteSpan = document.createElement('span'); this.deleteButton = document.createElement('button'); this.deleteButton.classList.add('mat-icon-button', 'mat-button-base', 'cell-icon'); this.deleteButton.setAttribute('style', 'width: auto; height: 24px; display: flex; align-items: center'); - this.deleteButton.innerHTML = `Remove`; + this.deleteButton.innerHTML = `Remove`; this.deleteSpan.append(this.deleteButton); this.gui.append(this.selectSpan, this.deleteSpan); diff --git a/vripper-ui/src/app/grid-custom-cells/event-message-renderer.native.ts b/vripper-ui/src/app/grid-custom-cells/event-message-renderer.native.ts new file mode 100644 index 0000000..ca5c755 --- /dev/null +++ b/vripper-ui/src/app/grid-custom-cells/event-message-renderer.native.ts @@ -0,0 +1,50 @@ +import {GridApi, ICellRendererComp, ICellRendererParams} from 'ag-grid-community'; +import {EventLog} from '../domain/event.model'; +import {EventLogService} from '../services/event-log.service'; + +export class EventMessageRendererNative implements ICellRendererComp { + private gui: HTMLElement; + private viewSpan: HTMLSpanElement; + private viewButton: HTMLButtonElement; + private eventLogService: EventLogService; + private eventLog: EventLog; + private gridApi: GridApi; + private text: HTMLSpanElement; + + destroy(): void { + this.viewButton.removeEventListener('click', () => this.eventLogService.openDialog(this.eventLog.message)); + } + + getGui(): HTMLElement { + return this.gui; + } + + init(params: ICellRendererParams): void { + // @ts-ignore + this.eventLogService = params.eventLogService; + this.eventLog = params.node.data; + this.gridApi = params.api; + this.gui = document.createElement('div'); + this.gui.setAttribute('style', 'display: flex; justify-content: space-between;'); + this.text = document.createElement('span'); + this.text.classList.add('no-wrap'); + this.text.textContent = this.eventLog.message; + this.viewSpan = document.createElement('span'); + this.viewButton = document.createElement('button'); + this.viewButton.classList.add('mat-icon-button', 'mat-button-base', 'cell-icon'); + this.viewButton.setAttribute('style', 'width: auto; height: 24px; display: flex; align-items: center'); + this.viewButton.innerHTML = ` + View`; + this.viewSpan.append(this.viewButton); + + this.gui.append(this.text, this.viewSpan); + + this.viewButton.addEventListener('click', () => this.eventLogService.openDialog(this.eventLog.message)); + } + + refresh(params: ICellRendererParams): boolean { + this.eventLog = params.node.data; + this.text.textContent = this.eventLog.message; + return true; + } +} diff --git a/vripper-ui/src/app/grid-custom-cells/status-renderer.native.ts b/vripper-ui/src/app/grid-custom-cells/status-renderer.native.ts index bd6f71e..6a929c4 100644 --- a/vripper-ui/src/app/grid-custom-cells/status-renderer.native.ts +++ b/vripper-ui/src/app/grid-custom-cells/status-renderer.native.ts @@ -61,8 +61,13 @@ export class StatusRendererNative implements ICellRendererComp { case 'DOWNLOADING': return ` +`; + case 'PROCESSING': + return ` + `; case 'COMPLETE': + case 'DONE': return ` `; @@ -86,8 +91,10 @@ export class StatusRendererNative implements ICellRendererComp { case 'PENDING': return 'pending'; case 'DOWNLOADING': + case 'PROCESSING': return 'downloading'; case 'COMPLETE': + case 'DONE': return 'complete'; case 'ERROR': return 'error'; diff --git a/vripper-ui/src/app/home/home.component.html b/vripper-ui/src/app/home/home.component.html index 671ce56..12e10b5 100644 --- a/vripper-ui/src/app/home/home.component.html +++ b/vripper-ui/src/app/home/home.component.html @@ -1,15 +1,31 @@ - - + + + + Downloads + + photo_album + + Multi-post links - link + link + + + Event Log + + grid_on + + + + diff --git a/vripper-ui/src/app/home/home.component.scss b/vripper-ui/src/app/home/home.component.scss index 1154353..c9244f6 100644 --- a/vripper-ui/src/app/home/home.component.scss +++ b/vripper-ui/src/app/home/home.component.scss @@ -47,3 +47,7 @@ table { align-self: start; } } + +.icon { + margin-left: 5px; +} diff --git a/vripper-ui/src/app/home/home.component.ts b/vripper-ui/src/app/home/home.component.ts index c75fbcf..b5f78d8 100644 --- a/vripper-ui/src/app/home/home.component.ts +++ b/vripper-ui/src/app/home/home.component.ts @@ -6,6 +6,10 @@ import {ChangeDetectionStrategy, Component, NgZone, OnDestroy, OnInit} from '@an import {MatDialog} from '@angular/material/dialog'; import {MatSnackBar} from '@angular/material/snack-bar'; import {HttpClient} from '@angular/common/http'; +import {EventLogService} from '../services/event-log.service'; +import {PostsService} from '../services/posts.service'; +import {MatTabChangeEvent} from '@angular/material/tabs'; +import {HomeTabsService} from '../services/home-tabs.service'; @Component({ selector: 'app-home', @@ -23,7 +27,10 @@ export class HomeComponent implements OnInit, OnDestroy { private serverService: ServerService, private _snackBar: MatSnackBar, private ngZone: NgZone, - public linkCollectorService: LinkCollectorService + public linkCollectorService: LinkCollectorService, + public eventLogService: EventLogService, + public postsService: PostsService, + public homeTabsService: HomeTabsService ) { } @@ -53,4 +60,8 @@ export class HomeComponent implements OnInit, OnDestroy { this.clipboardSub.unsubscribe(); } } + + onTabChange($event: MatTabChangeEvent) { + this.homeTabsService.setIndex($event.index); + } } diff --git a/vripper-ui/src/app/multi-post-grid/multi-post-grid-data.source.ts b/vripper-ui/src/app/multi-post-grid/multi-post-grid-data.source.ts index ecc713b..3c47cec 100644 --- a/vripper-ui/src/app/multi-post-grid/multi-post-grid-data.source.ts +++ b/vripper-ui/src/app/multi-post-grid/multi-post-grid-data.source.ts @@ -47,6 +47,7 @@ export class MultiPostGridDataSource { } disconnect() { + console.log('Disconnecting from link collector datasource'); this.subscriptions.forEach(e => e.unsubscribe()); } } diff --git a/vripper-ui/src/app/multi-post-grid/multi-post-grid.component.ts b/vripper-ui/src/app/multi-post-grid/multi-post-grid.component.ts index 9f482ba..f7c1fee 100644 --- a/vripper-ui/src/app/multi-post-grid/multi-post-grid.component.ts +++ b/vripper-ui/src/app/multi-post-grid/multi-post-grid.component.ts @@ -67,14 +67,18 @@ export class MultiPostGridComponent implements OnDestroy { overlayNoRowsTemplate: '', getRowNodeId: data => data['threadId'], onGridReady: () => { + this.linkCollectorService.setGridApi(this.gridOptions.api); this.dataSource = new MultiPostGridDataSource(this.wsConnection, this.gridOptions, this.zone); this.dataSource.connect(); }, onRowDataUpdated: () => this.linkCollectorService.setCount(this.gridOptions.api.getDisplayedRowCount()), + onRowDataChanged: () => this.linkCollectorService.setCount(this.gridOptions.api.getDisplayedRowCount()), }; } ngOnDestroy(): void { - this.dataSource.disconnect(); + if (this.dataSource) { + this.dataSource.disconnect(); + } } } diff --git a/vripper-ui/src/app/posts/posts.component.ts b/vripper-ui/src/app/posts/posts.component.ts index d6baf3f..69eeec5 100644 --- a/vripper-ui/src/app/posts/posts.component.ts +++ b/vripper-ui/src/app/posts/posts.component.ts @@ -3,7 +3,6 @@ import {ChangeDetectionStrategy, Component, NgZone, OnDestroy} from '@angular/co import {PostsDataSource} from './posts.datasource'; import {WsConnectionService} from '../services/ws-connection.service'; import {GridOptions} from 'ag-grid-community'; -import {Subject} from 'rxjs'; import {PostContextMenuService} from '../services/post-context-menu.service'; import {PostProgressRendererNative} from '../grid-custom-cells/post-progress-renderer.native'; import {PostStatusRendererNative} from '../grid-custom-cells/post-status-renderer.native'; @@ -19,7 +18,6 @@ import {PostsService} from '../services/posts.service'; changeDetection: ChangeDetectionStrategy.OnPush }) export class PostsComponent implements OnDestroy { - dialogOpen: Subject = new Subject(); gridOptions: GridOptions; dataSource: PostsDataSource; @@ -30,7 +28,8 @@ export class PostsComponent implements OnDestroy { private postsDataService: PostsService, private contextMenuService: PostContextMenuService, private overlayPositionBuilder: OverlayPositionBuilder, - private overlay: Overlay + private overlay: Overlay, + private postsService: PostsService ) { this.gridOptions = { columnDefs: [ @@ -110,11 +109,15 @@ export class PostsComponent implements OnDestroy { this.dataSource.connect(); }, onSelectionChanged: () => this.selectionService.onSelectionChanged(this.gridOptions.api.getSelectedNodes()), - onBodyScroll: () => this.contextMenuService.closePostContextMenu() + onBodyScroll: () => this.contextMenuService.closePostContextMenu(), + onRowDataUpdated: () => this.postsService.setCount(this.gridOptions.api.getDisplayedRowCount()), + onRowDataChanged: () => this.postsService.setCount(this.gridOptions.api.getDisplayedRowCount()), }; } ngOnDestroy(): void { - this.dataSource.disconnect(); + if (this.dataSource) { + this.dataSource.disconnect(); + } } } diff --git a/vripper-ui/src/app/services/event-log.service.ts b/vripper-ui/src/app/services/event-log.service.ts new file mode 100644 index 0000000..d97824a --- /dev/null +++ b/vripper-ui/src/app/services/event-log.service.ts @@ -0,0 +1,53 @@ +import {Injectable} from '@angular/core'; +import {BehaviorSubject, Observable, Subject} from 'rxjs'; +import {GridApi} from 'ag-grid-community'; +import {MatDialog, MatDialogConfig} from '@angular/material/dialog'; +import {EventLogMessageDialogComponent} from '../event-log/message-dialog/event-log-message-dialog.component'; + +@Injectable({ + providedIn: 'root' +}) +export class EventLogService { + private api: GridApi; + + private _count$: Subject = new BehaviorSubject(0); + + constructor(public dialog: MatDialog) { + } + + get count(): Observable { + return this._count$.asObservable(); + } + + setCount(count: number) { + this._count$.next(count); + } + + public setGridApi(api: GridApi) { + this.api = api; + } + + search(event) { + if (this.api) { + this.api.setQuickFilter(event); + } + } + + clear() { + if (this.api) { + this.api.setRowData([]); + } + } + + openDialog(message: string) { + console.log(message); + const config: MatDialogConfig = { + width: '70%', + height: '70%', + maxWidth: '100vw', + maxHeight: '100vh', + data: message + }; + this.dialog.open(EventLogMessageDialogComponent, config); + } +} diff --git a/vripper-ui/src/app/services/home-tabs.service.ts b/vripper-ui/src/app/services/home-tabs.service.ts new file mode 100644 index 0000000..3966074 --- /dev/null +++ b/vripper-ui/src/app/services/home-tabs.service.ts @@ -0,0 +1,18 @@ +import {Injectable} from '@angular/core'; +import {BehaviorSubject, Observable, Subject} from 'rxjs'; + +@Injectable({ + providedIn: 'root' +}) +export class HomeTabsService { + + private _index$: Subject = new BehaviorSubject(0); + + get index(): Observable { + return this._index$.asObservable(); + } + + setIndex(count: number) { + this._index$.next(count); + } +} diff --git a/vripper-ui/src/app/services/link-collector.service.ts b/vripper-ui/src/app/services/link-collector.service.ts index b89d39c..11c2a25 100644 --- a/vripper-ui/src/app/services/link-collector.service.ts +++ b/vripper-ui/src/app/services/link-collector.service.ts @@ -1,10 +1,12 @@ import {Injectable} from '@angular/core'; import {BehaviorSubject, Observable, Subject} from 'rxjs'; +import {GridApi} from 'ag-grid-community'; @Injectable({ providedIn: 'root' }) export class LinkCollectorService { + private api: GridApi; private _count$: Subject = new BehaviorSubject(0); @@ -12,7 +14,23 @@ export class LinkCollectorService { return this._count$.asObservable(); } + public setGridApi(api: GridApi) { + this.api = api; + } + + search(event) { + if (this.api) { + this.api.setQuickFilter(event); + } + } + setCount(count: number) { this._count$.next(count); } + + clear() { + if (this.api) { + this.api.setRowData([]); + } + } } diff --git a/vripper-ui/src/app/services/posts.service.ts b/vripper-ui/src/app/services/posts.service.ts index 39156b9..c64f557 100644 --- a/vripper-ui/src/app/services/posts.service.ts +++ b/vripper-ui/src/app/services/posts.service.ts @@ -1,5 +1,6 @@ import {Injectable} from '@angular/core'; import {GridApi} from 'ag-grid-community'; +import {BehaviorSubject, Observable, Subject} from 'rxjs'; @Injectable({ providedIn: 'root' @@ -7,6 +8,16 @@ import {GridApi} from 'ag-grid-community'; export class PostsService { private api: GridApi; + private _count$: Subject = new BehaviorSubject(0); + + get count(): Observable { + return this._count$.asObservable(); + } + + setCount(count: number) { + this._count$.next(count); + } + public setGridApi(api: GridApi) { this.api = api; } @@ -23,6 +34,8 @@ export class PostsService { } search(event) { - this.api.setQuickFilter(event); + if (this.api) { + this.api.setQuickFilter(event); + } } } diff --git a/vripper-ui/src/app/services/ws-connection.service.ts b/vripper-ui/src/app/services/ws-connection.service.ts index 38ea2cf..6305192 100644 --- a/vripper-ui/src/app/services/ws-connection.service.ts +++ b/vripper-ui/src/app/services/ws-connection.service.ts @@ -12,6 +12,7 @@ import {GlobalState} from '../domain/global-state.model'; import {map, share} from 'rxjs/operators'; import {RxStomp, RxStompConfig, RxStompState} from '@stomp/rx-stomp'; import {ElectronService} from 'ngx-electron'; +import {EventLog} from '../domain/event.model'; @Injectable({ providedIn: 'root' @@ -22,10 +23,12 @@ export class WsConnectionService { private speed: Observable; private user: Observable; private posts: Observable>; + private events: Observable>; private postDetails: Observable>; private multiPostModels: Observable>; private queuedRemove: Observable>; private postsRemove: Observable>; + private eventsRemove: Observable>; private postIdDetails: string; private connectionState$: Subject = new BehaviorSubject(RxStompState.CLOSED); @@ -59,6 +62,14 @@ export class WsConnectionService { return this.postsRemove; } + get events$(): Observable { + return this.events; + } + + get eventsRemove$(): Observable { + return this.eventsRemove; + } + get queuedRemove$(): Observable { return this.queuedRemove; } @@ -121,6 +132,13 @@ export class WsConnectionService { share() ); + this.eventsRemove = this.rxStomp.watch('/topic/events/deleted').pipe( + map(e => { + return JSON.parse(e.body); + }), + share() + ); + this.queuedRemove = this.rxStomp.watch('/topic/queued/deleted').pipe( map(e => { return JSON.parse(e.body); @@ -176,6 +194,25 @@ export class WsConnectionService { share() ); + this.events = this.rxStomp.watch('/topic/events').pipe( + map(e => { + const events: Array = []; + (>JSON.parse(e.body)).forEach(element => { + events.push( + new EventLog( + element.id, + element.type, + element.status, + element.time, + element.message + ) + ); + }); + return events; + }), + share() + ); + this.multiPostModels = this.rxStomp.watch('/topic/queued').pipe( map(e => { const multiPostModels: Array = []; diff --git a/vripper-ui/src/app/settings/settings.component.html b/vripper-ui/src/app/settings/settings.component.html index fad43b9..36ae3d1 100644 --- a/vripper-ui/src/app/settings/settings.component.html +++ b/vripper-ui/src/app/settings/settings.component.html @@ -134,6 +134,13 @@
+ + Select a proxy + + {{mirror}} + + + Enable ViperGirls Authentication @@ -169,6 +176,28 @@
+ +
+
+ + + + + +
+
+
@@ -188,11 +217,12 @@ - - - - -
-
-
- - + + + + + +
+
+
+ + +
+ + + + + +