12 Commits

Author SHA1 Message Date
Yuvi9587
5a8c151c97 Deviant Support fix 2025-12-23 22:52:50 +05:30
Yuvi9587
50ba60a461 Fixed devient download 2025-12-23 21:27:21 +05:30
Yuvi9587
23521e7060 Added "Proxy/Network" Tab 2025-12-23 21:27:08 +05:30
Yuvi9587
f9c504b936 Proxy Support 2025-12-23 21:26:49 +05:30
Yuvi9587
efa0abd0f1 Fixed devient download (Kinda) 2025-12-23 21:26:34 +05:30
Yuvi9587
7d76d00470 Proxy 2025-12-23 21:26:18 +05:30
Yuvi9587
1494d3f456 Proxy Support Keys 2025-12-23 21:26:11 +05:30
Yuvi9587
675646e763 Fixed Error Dialog 2025-12-22 09:15:26 +05:30
Yuvi9587
611e892576 "add to queue" button 2025-12-21 22:12:44 +05:30
Yuvi9587
23fd7f0714 Added a "add to queue" feature 2025-12-21 22:12:34 +05:30
Yuvi9587
cfcd800a49 Fixed unnecessary fetch in renaming mode 2025-12-21 22:12:14 +05:30
Yuvi9587
24acec2dc3 Fixed unnecessary fetch in renaming mode 2025-12-21 22:12:09 +05:30
9 changed files with 746 additions and 234 deletions

View File

@@ -68,6 +68,14 @@ DISCORD_TOKEN_KEY = "discord/token"
POST_DOWNLOAD_ACTION_KEY = "postDownloadAction" POST_DOWNLOAD_ACTION_KEY = "postDownloadAction"
# --- Proxy / Network Keys ---
PROXY_ENABLED_KEY = "proxy/enabled"
PROXY_HOST_KEY = "proxy/host"
PROXY_PORT_KEY = "proxy/port"
PROXY_USERNAME_KEY = "proxy/username"
PROXY_PASSWORD_KEY = "proxy/password"
# --- UI Constants and Identifiers --- # --- UI Constants and Identifiers ---
HTML_PREFIX = "<!HTML!>" HTML_PREFIX = "<!HTML!>"
LOG_DISPLAY_LINKS = "links" LOG_DISPLAY_LINKS = "links"

View File

@@ -6,11 +6,13 @@ import requests
import cloudscraper import cloudscraper
from ..utils.network_utils import extract_post_info, prepare_cookies_for_request from ..utils.network_utils import extract_post_info, prepare_cookies_for_request
from ..config.constants import ( from ..config.constants import (
STYLE_DATE_POST_TITLE STYLE_DATE_POST_TITLE,
STYLE_DATE_BASED,
STYLE_POST_TITLE_GLOBAL_NUMBERING
) )
def fetch_posts_paginated(api_url_base, headers, offset, logger, cancellation_event=None, pause_event=None, cookies_dict=None): def fetch_posts_paginated(api_url_base, headers, offset, logger, cancellation_event=None, pause_event=None, cookies_dict=None, proxies=None):
""" """
Fetches a single page of posts from the API with robust retry logic. Fetches a single page of posts from the API with robust retry logic.
""" """
@@ -39,7 +41,7 @@ def fetch_posts_paginated(api_url_base, headers, offset, logger, cancellation_ev
logger(log_message) logger(log_message)
try: try:
with requests.get(paginated_url, headers=headers, timeout=(15, 60), cookies=cookies_dict) as response: with requests.get(paginated_url, headers=headers, timeout=(15, 60), cookies=cookies_dict, proxies=proxies) as response:
response.raise_for_status() response.raise_for_status()
response.encoding = 'utf-8' response.encoding = 'utf-8'
return response.json() return response.json()
@@ -79,9 +81,8 @@ def fetch_posts_paginated(api_url_base, headers, offset, logger, cancellation_ev
raise RuntimeError(f"Failed to fetch page {paginated_url} after all attempts.") raise RuntimeError(f"Failed to fetch page {paginated_url} after all attempts.")
def fetch_single_post_data(api_domain, service, user_id, post_id, headers, logger, cookies_dict=None): def fetch_single_post_data(api_domain, service, user_id, post_id, headers, logger, cookies_dict=None, proxies=None):
""" """
--- MODIFIED FUNCTION ---
Fetches the full data, including the 'content' field, for a single post using cloudscraper. Fetches the full data, including the 'content' field, for a single post using cloudscraper.
""" """
post_api_url = f"https://{api_domain}/api/v1/{service}/user/{user_id}/post/{post_id}" post_api_url = f"https://{api_domain}/api/v1/{service}/user/{user_id}/post/{post_id}"
@@ -91,7 +92,7 @@ def fetch_single_post_data(api_domain, service, user_id, post_id, headers, logge
scraper = None scraper = None
try: try:
scraper = cloudscraper.create_scraper() scraper = cloudscraper.create_scraper()
response = scraper.get(post_api_url, headers=headers, timeout=(15, 300), cookies=cookies_dict) response = scraper.get(post_api_url, headers=headers, timeout=(15, 300), cookies=cookies_dict, proxies=proxies)
response.raise_for_status() response.raise_for_status()
full_post_data = response.json() full_post_data = response.json()
@@ -106,12 +107,11 @@ def fetch_single_post_data(api_domain, service, user_id, post_id, headers, logge
logger(f" ❌ Failed to fetch full content for post {post_id}: {e}") logger(f" ❌ Failed to fetch full content for post {post_id}: {e}")
return None return None
finally: finally:
# CRITICAL FIX: Close the scraper session to free file descriptors and memory
if scraper: if scraper:
scraper.close() scraper.close()
def fetch_post_comments(api_domain, service, user_id, post_id, headers, logger, cancellation_event=None, pause_event=None, cookies_dict=None): def fetch_post_comments(api_domain, service, user_id, post_id, headers, logger, cancellation_event=None, pause_event=None, cookies_dict=None, proxies=None):
"""Fetches all comments for a specific post.""" """Fetches all comments for a specific post."""
if cancellation_event and cancellation_event.is_set(): if cancellation_event and cancellation_event.is_set():
raise RuntimeError("Comment fetch operation cancelled by user.") raise RuntimeError("Comment fetch operation cancelled by user.")
@@ -120,8 +120,7 @@ def fetch_post_comments(api_domain, service, user_id, post_id, headers, logger,
logger(f" Fetching comments: {comments_api_url}") logger(f" Fetching comments: {comments_api_url}")
try: try:
# FIX: Use context manager with requests.get(comments_api_url, headers=headers, timeout=(10, 30), cookies=cookies_dict, proxies=proxies) as response:
with requests.get(comments_api_url, headers=headers, timeout=(10, 30), cookies=cookies_dict) as response:
response.raise_for_status() response.raise_for_status()
response.encoding = 'utf-8' response.encoding = 'utf-8'
return response.json() return response.json()
@@ -144,7 +143,8 @@ def download_from_api(
app_base_dir=None, app_base_dir=None,
manga_filename_style_for_sort_check=None, manga_filename_style_for_sort_check=None,
processed_post_ids=None, processed_post_ids=None,
fetch_all_first=False fetch_all_first=False,
proxies=None
): ):
parsed_input_url_for_domain = urlparse(api_url_input) parsed_input_url_for_domain = urlparse(api_url_input)
api_domain = parsed_input_url_for_domain.netloc api_domain = parsed_input_url_for_domain.netloc
@@ -180,8 +180,7 @@ def download_from_api(
direct_post_api_url = f"https://{api_domain}/api/v1/{service}/user/{user_id}/post/{target_post_id}" direct_post_api_url = f"https://{api_domain}/api/v1/{service}/user/{user_id}/post/{target_post_id}"
logger(f" Attempting direct fetch for target post: {direct_post_api_url}") logger(f" Attempting direct fetch for target post: {direct_post_api_url}")
try: try:
# FIX: Use context manager with requests.get(direct_post_api_url, headers=headers, timeout=(10, 30), cookies=cookies_for_api, proxies=proxies) as direct_response:
with requests.get(direct_post_api_url, headers=headers, timeout=(10, 30), cookies=cookies_for_api) as direct_response:
direct_response.raise_for_status() direct_response.raise_for_status()
direct_response.encoding = 'utf-8' direct_response.encoding = 'utf-8'
direct_post_data = direct_response.json() direct_post_data = direct_response.json()
@@ -208,12 +207,23 @@ def download_from_api(
if target_post_id and (start_page or end_page): if target_post_id and (start_page or end_page):
logger("⚠️ Page range (start/end page) is ignored when a specific post URL is provided (searching all pages for the post).") logger("⚠️ Page range (start/end page) is ignored when a specific post URL is provided (searching all pages for the post).")
is_manga_mode_fetch_all_and_sort_oldest_first = manga_mode and (manga_filename_style_for_sort_check != STYLE_DATE_POST_TITLE) and not target_post_id # --- FIXED LOGIC HERE ---
# Define which styles require fetching ALL posts first (Sequential Mode)
styles_requiring_fetch_all = [STYLE_DATE_BASED, STYLE_POST_TITLE_GLOBAL_NUMBERING]
# Only enable "fetch all and sort" if the current style is explicitly in the list above
is_manga_mode_fetch_all_and_sort_oldest_first = (
manga_mode and
(manga_filename_style_for_sort_check in styles_requiring_fetch_all) and
not target_post_id
)
should_fetch_all = fetch_all_first or is_manga_mode_fetch_all_and_sort_oldest_first should_fetch_all = fetch_all_first or is_manga_mode_fetch_all_and_sort_oldest_first
api_base_url = f"https://{api_domain}/api/v1/{service}/user/{user_id}/posts" api_base_url = f"https://{api_domain}/api/v1/{service}/user/{user_id}/posts"
page_size = 50 page_size = 50
if is_manga_mode_fetch_all_and_sort_oldest_first: if is_manga_mode_fetch_all_and_sort_oldest_first:
logger(f" Manga Mode (Style: {manga_filename_style_for_sort_check if manga_filename_style_for_sort_check else 'Default'} - Oldest First Sort Active): Fetching all posts to sort by date...") logger(f" Manga Mode (Style: {manga_filename_style_for_sort_check} - Oldest First Sort Active): Fetching all posts to sort by date...")
all_posts_for_manga_mode = [] all_posts_for_manga_mode = []
current_offset_manga = 0 current_offset_manga = 0
if start_page and start_page > 1: if start_page and start_page > 1:
@@ -240,7 +250,7 @@ def download_from_api(
logger(f" Manga Mode: Reached specified end page ({end_page}). Stopping post fetch.") logger(f" Manga Mode: Reached specified end page ({end_page}). Stopping post fetch.")
break break
try: try:
posts_batch_manga = fetch_posts_paginated(api_base_url, headers, current_offset_manga, logger, cancellation_event, pause_event, cookies_dict=cookies_for_api) posts_batch_manga = fetch_posts_paginated(api_base_url, headers, current_offset_manga, logger, cancellation_event, pause_event, cookies_dict=cookies_for_api, proxies=proxies)
if not isinstance(posts_batch_manga, list): if not isinstance(posts_batch_manga, list):
logger(f"❌ API Error (Manga Mode): Expected list of posts, got {type(posts_batch_manga)}.") logger(f"❌ API Error (Manga Mode): Expected list of posts, got {type(posts_batch_manga)}.")
break break
@@ -308,8 +318,9 @@ def download_from_api(
yield all_posts_for_manga_mode[i:i + page_size] yield all_posts_for_manga_mode[i:i + page_size]
return return
if manga_mode and not target_post_id and (manga_filename_style_for_sort_check == STYLE_DATE_POST_TITLE): # Log specific message for styles that are in Manga Mode but NOT sorting (Streaming)
logger(f" Manga Mode (Style: {STYLE_DATE_POST_TITLE}): Processing posts in default API order (newest first).") if manga_mode and not target_post_id and (manga_filename_style_for_sort_check not in styles_requiring_fetch_all):
logger(f" Renaming Mode (Style: {manga_filename_style_for_sort_check}): Processing posts in default API order (Streaming).")
current_page_num = 1 current_page_num = 1
current_offset = 0 current_offset = 0
@@ -341,7 +352,7 @@ def download_from_api(
break break
try: try:
raw_posts_batch = fetch_posts_paginated(api_base_url, headers, current_offset, logger, cancellation_event, pause_event, cookies_dict=cookies_for_api) raw_posts_batch = fetch_posts_paginated(api_base_url, headers, current_offset, logger, cancellation_event, pause_event, cookies_dict=cookies_for_api, proxies=proxies)
if not isinstance(raw_posts_batch, list): if not isinstance(raw_posts_batch, list):
logger(f"❌ API Error: Expected list of posts, got {type(raw_posts_batch)} at page {current_page_num} (offset {current_offset}).") logger(f"❌ API Error: Expected list of posts, got {type(raw_posts_batch)} at page {current_page_num} (offset {current_offset}).")
break break

View File

@@ -13,9 +13,17 @@ class DeviantArtClient:
def __init__(self, logger_func=print): def __init__(self, logger_func=print):
self.session = requests.Session() self.session = requests.Session()
# Headers matching 1.py (Firefox)
self.session.headers.update({ self.session.headers.update({
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0",
'Accept': '*/*', "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.5",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
}) })
self.access_token = None self.access_token = None
self.logger = logger_func self.logger = logger_func
@@ -59,8 +67,20 @@ class DeviantArtClient:
while True: while True:
try: try:
resp = self.session.get(url, params=params, timeout=20) resp = self.session.get(url, params=params, timeout=20)
# Handle Token Expiration (401) # 429: Rate Limit (Retry infinitely like 1.py)
if resp.status_code == 429:
retry_after = resp.headers.get('Retry-After')
if retry_after:
sleep_time = int(retry_after) + 1
else:
sleep_time = 5 # Default sleep from 1.py
self._log_once(sleep_time, f" [DeviantArt] ⚠️ Rate limit (429). Sleeping {sleep_time}s...")
time.sleep(sleep_time)
continue
# 401: Token Expired (Refresh and Retry)
if resp.status_code == 401: if resp.status_code == 401:
self.logger(" [DeviantArt] Token expired. Refreshing...") self.logger(" [DeviantArt] Token expired. Refreshing...")
if self.authenticate(): if self.authenticate():
@@ -69,59 +89,44 @@ class DeviantArtClient:
else: else:
raise Exception("Failed to refresh token") raise Exception("Failed to refresh token")
# Handle Rate Limiting (429) if 400 <= resp.status_code < 500:
if resp.status_code == 429: resp.raise_for_status() # This raises immediately, breaking the loop
if retries < max_retries:
retry_after = resp.headers.get('Retry-After') if 500 <= resp.status_code < 600:
resp.raise_for_status()
if retry_after:
sleep_time = int(retry_after) + 1
msg = f" [DeviantArt] ⚠️ Rate limit (Server says wait {sleep_time}s)."
else:
sleep_time = backoff_delay * (2 ** retries)
msg = f" [DeviantArt] ⚠️ Rate limit reached. Retrying in {sleep_time}s..."
# --- THREAD-SAFE LOGGING CHECK ---
should_log = False
with self.log_lock:
if sleep_time not in self.logged_waits:
self.logged_waits.add(sleep_time)
should_log = True
if should_log:
self.logger(msg)
time.sleep(sleep_time)
retries += 1
continue
else:
resp.raise_for_status()
resp.raise_for_status() resp.raise_for_status()
# Clear log history on success so we get warned again if limits return later
with self.log_lock: with self.log_lock:
if self.logged_waits: self.logged_waits.clear()
self.logged_waits.clear()
return resp.json() return resp.json()
except requests.exceptions.HTTPError as e:
if e.response is not None and 400 <= e.response.status_code < 500:
raise e
# Otherwise fall through to general retry logic (for 5xx)
pass
except requests.exceptions.RequestException as e: except requests.exceptions.RequestException as e:
# Network errors / 5xx errors -> Retry
if retries < max_retries: if retries < max_retries:
# Using the lock here too to prevent connection error spam self._log_once("conn_error", f" [DeviantArt] Connection error: {e}. Retrying...")
should_log = False time.sleep(backoff_delay)
with self.log_lock:
if "conn_error" not in self.logged_waits:
self.logged_waits.add("conn_error")
should_log = True
if should_log:
self.logger(f" [DeviantArt] Connection error: {e}. Retrying...")
time.sleep(2)
retries += 1 retries += 1
continue continue
raise e raise e
def _log_once(self, key, message):
"""Helper to avoid spamming the same log message during loops."""
should_log = False
with self.log_lock:
if key not in self.logged_waits:
self.logged_waits.add(key)
should_log = True
if should_log:
self.logger(message)
def get_deviation_uuid(self, url): def get_deviation_uuid(self, url):
"""Scrapes the deviation page to find the UUID.""" """Scrapes the deviation page to find the UUID."""
@@ -139,13 +144,17 @@ class DeviantArtClient:
def get_deviation_content(self, uuid): def get_deviation_content(self, uuid):
"""Fetches download info.""" """Fetches download info."""
# 1. Try high-res download endpoint
try: try:
data = self._api_call(f"/deviation/download/{uuid}") data = self._api_call(f"/deviation/download/{uuid}")
if 'src' in data: if 'src' in data:
return data return data
except: except:
# If 400/403 (Not downloadable), we fail silently here
# and proceed to step 2 (Metadata fallback)
pass pass
# 2. Fallback to standard content
try: try:
meta = self._api_call(f"/deviation/{uuid}") meta = self._api_call(f"/deviation/{uuid}")
if 'content' in meta: if 'content' in meta:

View File

@@ -3,7 +3,7 @@ import time
import os import os
import json import json
import traceback import traceback
from concurrent.futures import ThreadPoolExecutor, as_completed, Future from concurrent.futures import ThreadPoolExecutor, as_completed, Future, CancelledError
from .api_client import download_from_api from .api_client import download_from_api
from .workers import PostProcessorWorker from .workers import PostProcessorWorker
from ..config.constants import ( from ..config.constants import (
@@ -84,8 +84,18 @@ class DownloadManager:
is_single_post = bool(config.get('target_post_id_from_initial_url')) is_single_post = bool(config.get('target_post_id_from_initial_url'))
use_multithreading = config.get('use_multithreading', True) use_multithreading = config.get('use_multithreading', True)
is_manga_sequential = config.get('manga_mode_active') and config.get('manga_filename_style') in [STYLE_DATE_BASED, STYLE_POST_TITLE_GLOBAL_NUMBERING]
# --- FIXED LOGIC: Strict check for sequential fetch modes ---
# Only "Date Based" and "Title + Global Numbering" require fetching the full list first.
# "Custom", "Date + Title", "Original Name", and "Post ID" will now use the pool (streaming).
sequential_styles = [STYLE_DATE_BASED, STYLE_POST_TITLE_GLOBAL_NUMBERING]
is_manga_sequential = (
config.get('manga_mode_active') and
config.get('manga_filename_style') in sequential_styles
)
# If it is NOT a strictly sequential manga mode, we use the pool (fetch-as-we-go)
should_use_multithreading_for_posts = use_multithreading and not is_single_post and not is_manga_sequential should_use_multithreading_for_posts = use_multithreading and not is_single_post and not is_manga_sequential
if should_use_multithreading_for_posts: if should_use_multithreading_for_posts:
@@ -97,12 +107,34 @@ class DownloadManager:
fetcher_thread.start() fetcher_thread.start()
else: else:
# Single-threaded mode does not use the manager's complex logic # Single-threaded mode does not use the manager's complex logic
self._log(" Manager is handing off to a single-threaded worker...") self._log(" Manager is handing off to a single-threaded worker (Sequential Mode)...")
# The single-threaded worker will manage its own lifecycle and signals. # The single-threaded worker will manage its own lifecycle and signals.
# The manager's role for this session is effectively over. # The manager's role for this session is effectively over.
self.is_running = False # Allow another session to start if needed self.is_running = False # Allow another session to start if needed
self.progress_queue.put({'type': 'handoff_to_single_thread', 'payload': (config,)}) self.progress_queue.put({'type': 'handoff_to_single_thread', 'payload': (config,)})
def _get_proxies_from_config(self, config):
"""Constructs the proxy dictionary from the config."""
if not config.get('proxy_enabled'):
return None
host = config.get('proxy_host')
port = config.get('proxy_port')
if not host or not port:
return None
proxy_str = f"http://{host}:{port}"
# Add auth if provided
user = config.get('proxy_username')
password = config.get('proxy_password')
if user and password:
proxy_str = f"http://{user}:{password}@{host}:{port}"
return {
"http": proxy_str,
"https": proxy_str
}
def _fetch_and_queue_posts_for_pool(self, config, restore_data, creator_profile_data): def _fetch_and_queue_posts_for_pool(self, config, restore_data, creator_profile_data):
""" """
@@ -117,6 +149,9 @@ class DownloadManager:
session_processed_ids = set(restore_data.get('processed_post_ids', [])) if restore_data else set() session_processed_ids = set(restore_data.get('processed_post_ids', [])) if restore_data else set()
profile_processed_ids = set(creator_profile_data.get('processed_post_ids', [])) profile_processed_ids = set(creator_profile_data.get('processed_post_ids', []))
processed_ids = session_processed_ids.union(profile_processed_ids) processed_ids = session_processed_ids.union(profile_processed_ids)
# Helper to get proxies
proxies = self._get_proxies_from_config(config)
if restore_data and 'all_posts_data' in restore_data: if restore_data and 'all_posts_data' in restore_data:
# This logic for session restore remains as it relies on a pre-fetched list # This logic for session restore remains as it relies on a pre-fetched list
@@ -132,127 +167,113 @@ class DownloadManager:
return return
for post_data in posts_to_process: for post_data in posts_to_process:
if self.cancellation_event.is_set(): break if self.cancellation_event.is_set():
worker = PostProcessorWorker(post_data, config, self.progress_queue) break
worker_args = self._map_config_to_worker_args(post_data, config)
# Manually inject proxies here if _map_config_to_worker_args didn't catch it (though it should)
worker_args['proxies'] = proxies
worker = PostProcessorWorker(**worker_args)
future = self.thread_pool.submit(worker.process) future = self.thread_pool.submit(worker.process)
future.add_done_callback(self._handle_future_result) future.add_done_callback(self._handle_future_result)
self.active_futures.append(future) self.active_futures.append(future)
else: else:
# --- START: REFACTORED STREAMING LOGIC --- # --- Streaming Logic ---
if proxies:
self._log(f" 🌐 Using Proxy: {config.get('proxy_host')}:{config.get('proxy_port')}")
post_generator = download_from_api( post_generator = download_from_api(
api_url_input=config['api_url'], api_url_input=config['api_url'],
logger=self._log, logger=self._log,
start_page=config.get('start_page'), start_page=config.get('start_page'),
end_page=config.get('end_page'), end_page=config.get('end_page'),
manga_mode=config.get('manga_mode_active', False),
cancellation_event=self.cancellation_event, cancellation_event=self.cancellation_event,
pause_event=self.pause_event, pause_event=self.pause_event,
use_cookie=config.get('use_cookie', False), cookies_dict=None, # Cookie handling handled inside client if needed
cookie_text=config.get('cookie_text', ''), proxies=proxies # <--- NEW: Pass proxies to API client
selected_cookie_file=config.get('selected_cookie_file'),
app_base_dir=config.get('app_base_dir'),
manga_filename_style_for_sort_check=config.get('manga_filename_style'),
processed_post_ids=list(processed_ids)
) )
self.total_posts = 0 for post_batch in post_generator:
self.processed_posts = 0
# Process posts in batches as they are yielded by the API client
for batch in post_generator:
if self.cancellation_event.is_set(): if self.cancellation_event.is_set():
self._log(" Post fetching cancelled.")
break break
# Filter out any posts that might have been processed since the start if not post_batch:
posts_in_batch_to_process = [p for p in batch if p.get('id') not in processed_ids]
if not posts_in_batch_to_process:
continue continue
# Update total count and immediately inform the UI new_posts_batch = [p for p in post_batch if p.get('id') not in processed_ids]
self.total_posts += len(posts_in_batch_to_process)
self.progress_queue.put({'type': 'overall_progress', 'payload': (self.total_posts, self.processed_posts)}) if not new_posts_batch:
continue
for post_data in posts_in_batch_to_process: # Update total posts dynamically as we find them
if self.cancellation_event.is_set(): break self.total_posts += len(new_posts_batch)
worker = PostProcessorWorker(post_data, config, self.progress_queue)
for post_data in new_posts_batch:
if self.cancellation_event.is_set():
break
# MAPPING CONFIG TO WORKER ARGS
worker_args = self._map_config_to_worker_args(post_data, config)
worker = PostProcessorWorker(**worker_args)
future = self.thread_pool.submit(worker.process) future = self.thread_pool.submit(worker.process)
future.add_done_callback(self._handle_future_result) future.add_done_callback(self._handle_future_result)
self.active_futures.append(future) self.active_futures.append(future)
if self.total_posts == 0 and not self.cancellation_event.is_set(): # Small sleep to prevent UI freeze
self._log("✅ No new posts found to process.") time.sleep(0.01)
except Exception as e: except Exception as e:
self._log(f"❌ CRITICAL ERROR in post fetcher thread: {e}") self._log(f"❌ Critical Error in Fetcher Thread: {e}")
self._log(traceback.format_exc()) traceback.print_exc()
finally: finally:
if self.thread_pool: self.is_running = False # Mark as not running so we can finish
self.thread_pool.shutdown(wait=True) # The main window checks active futures, so we just exit this thread.
self.is_running = False
self._log("🏁 All processing tasks have completed or been cancelled.")
self.progress_queue.put({
'type': 'finished',
'payload': (self.total_downloads, self.total_skips, self.cancellation_event.is_set(), self.all_kept_original_filenames)
})
def _handle_future_result(self, future: Future): def _map_config_to_worker_args(self, post_data, config):
"""Callback executed when a worker task completes.""" """Helper to map the flat config dict to PostProcessorWorker arguments."""
if self.cancellation_event.is_set(): # Get proxy dict
return proxies = self._get_proxies_from_config(config)
with threading.Lock(): # Protect shared counters
self.processed_posts += 1
try:
if future.cancelled():
self._log("⚠️ A post processing task was cancelled.")
self.total_skips += 1
else:
result = future.result()
(dl_count, skip_count, kept_originals,
retryable, permanent, history) = result
self.total_downloads += dl_count
self.total_skips += skip_count
self.all_kept_original_filenames.extend(kept_originals)
if retryable:
self.progress_queue.put({'type': 'retryable_failure', 'payload': (retryable,)})
if permanent:
self.progress_queue.put({'type': 'permanent_failure', 'payload': (permanent,)})
if history:
self.progress_queue.put({'type': 'post_processed_history', 'payload': (history,)})
post_id = history.get('post_id')
if post_id and self.current_creator_profile_path:
profile_data = self._setup_creator_profile({'creator_name_for_profile': self.current_creator_name_for_profile, 'session_file_path': self.session_file_path})
if post_id not in profile_data.get('processed_post_ids', []):
profile_data.setdefault('processed_post_ids', []).append(post_id)
self._save_creator_profile(profile_data)
except Exception as e:
self._log(f"❌ Worker task resulted in an exception: {e}")
self.total_skips += 1 # Count errored posts as skipped
self.progress_queue.put({'type': 'overall_progress', 'payload': (self.total_posts, self.processed_posts)})
# This mirrors the arguments in workers.py PostProcessorWorker.__init__
return {
'post_data': post_data,
'download_root': config.get('output_dir'),
'known_names': [], # If needed, pass KNOWN_NAMES or load them
'filter_character_list': [], # Parsed filters if available in config
'emitter': self.progress_queue,
'unwanted_keywords': set(), # Parse if needed
'filter_mode': config.get('filter_mode'),
'skip_zip': config.get('skip_zip'),
'use_subfolders': config.get('use_subfolders'),
'use_post_subfolders': config.get('use_post_subfolders'),
'target_post_id_from_initial_url': config.get('target_post_id_from_initial_url'),
'custom_folder_name': config.get('custom_folder_name'),
'compress_images': config.get('compress_images'),
'download_thumbnails': config.get('download_thumbnails'),
'service': config.get('service') or 'unknown',
'user_id': config.get('user_id') or 'unknown',
'pause_event': self.pause_event,
'api_url_input': config.get('api_url'),
'cancellation_event': self.cancellation_event,
'downloaded_files': None,
'downloaded_file_hashes': None,
'downloaded_files_lock': None,
'downloaded_file_hashes_lock': None,
'manga_mode_active': config.get('manga_mode_active'),
'manga_filename_style': config.get('manga_filename_style'),
'manga_custom_filename_format': config.get('custom_manga_filename_format', "{published} {title}"),
'manga_custom_date_format': config.get('manga_custom_date_format', "YYYY-MM-DD"),
'use_multithreading': config.get('use_multithreading', True),
'proxies': proxies, # <--- NEW: Pass proxies to worker
}
def _setup_creator_profile(self, config): def _setup_creator_profile(self, config):
"""Prepares the path and loads data for the current creator's profile.""" """Prepares the path and loads data for the current creator's profile."""
self.current_creator_name_for_profile = config.get('creator_name_for_profile') # Extract name logic here or assume config has it
if not self.current_creator_name_for_profile: self.current_creator_name_for_profile = "Unknown"
self._log("⚠️ Cannot create creator profile: Name not provided in config.") # You should ideally extract name from URL or config here if available
return {}
appdata_dir = os.path.dirname(config.get('session_file_path', '.'))
self.creator_profiles_dir = os.path.join(appdata_dir, "creator_profiles")
os.makedirs(self.creator_profiles_dir, exist_ok=True)
safe_filename = clean_folder_name(self.current_creator_name_for_profile) + ".json"
self.current_creator_profile_path = os.path.join(self.creator_profiles_dir, safe_filename)
if os.path.exists(self.current_creator_profile_path):
try:
with open(self.current_creator_profile_path, 'r', encoding='utf-8') as f:
return json.load(f)
except (json.JSONDecodeError, OSError) as e:
self._log(f"❌ Error loading creator profile '{safe_filename}': {e}. Starting fresh.")
return {} return {}
def _save_creator_profile(self, data): def _save_creator_profile(self, data):
@@ -280,6 +301,33 @@ class DownloadManager:
self.cancellation_event.set() self.cancellation_event.set()
if self.thread_pool: if self.thread_pool:
self._log(" Signaling all worker threads to stop and shutting down pool...") self.thread_pool.shutdown(wait=False, cancel_futures=True)
self.thread_pool.shutdown(wait=False)
def _handle_future_result(self, future):
"""Callback for when a worker task finishes."""
if self.active_futures:
try:
self.active_futures.remove(future)
except ValueError:
pass
try:
result = future.result()
# result tuple: (download_count, skip_count, kept_original_filenames, ...)
if result:
self.total_downloads += result[0]
self.total_skips += result[1]
if len(result) > 3 and result[3]:
# filename was kept original
pass
except CancelledError:
pass
except Exception as e:
self._log(f"❌ Worker Error: {e}")
self.processed_posts += 1
self.progress_queue.put({'type': 'overall_progress', 'payload': (self.total_posts, self.processed_posts)})
if not self.active_futures and not self.is_running:
self._log("✅ All tasks completed.")
self.progress_queue.put({'type': 'worker_finished', 'payload': (self.total_downloads, self.total_skips, [], [])})

View File

@@ -133,7 +133,8 @@ class PostProcessorWorker:
sfp_threshold=None, sfp_threshold=None,
handle_unknown_mode=False, handle_unknown_mode=False,
creator_name_cache=None, creator_name_cache=None,
add_info_in_pdf=False add_info_in_pdf=False,
proxies=None
): ):
self.post = post_data self.post = post_data
@@ -208,9 +209,8 @@ class PostProcessorWorker:
self.sfp_threshold = sfp_threshold self.sfp_threshold = sfp_threshold
self.handle_unknown_mode = handle_unknown_mode self.handle_unknown_mode = handle_unknown_mode
self.creator_name_cache = creator_name_cache self.creator_name_cache = creator_name_cache
#-- New assign --
self.add_info_in_pdf = add_info_in_pdf self.add_info_in_pdf = add_info_in_pdf
#-- New assign -- self.proxies = proxies
if self.compress_images and Image is None: if self.compress_images and Image is None:
@@ -263,7 +263,7 @@ class PostProcessorWorker:
new_url = parsed_url._replace(netloc=new_domain).geturl() new_url = parsed_url._replace(netloc=new_domain).geturl()
try: try:
with requests.head(new_url, headers={'User-Agent': 'Mozilla/5.0'}, timeout=5, allow_redirects=True) as resp: with requests.head(new_url, headers={'User-Agent': 'Mozilla/5.0'}, timeout=5, allow_redirects=True, proxies=self.proxies) as resp:
if resp.status_code == 200: if resp.status_code == 200:
return new_url return new_url
except requests.RequestException: except requests.RequestException:
@@ -338,7 +338,8 @@ class PostProcessorWorker:
api_original_filename_for_size_check = file_info.get('_original_name_for_log', file_info.get('name')) api_original_filename_for_size_check = file_info.get('_original_name_for_log', file_info.get('name'))
try: try:
# Use a stream=True HEAD request to get headers without downloading the body # Use a stream=True HEAD request to get headers without downloading the body
with requests.head(file_url, headers=file_download_headers, timeout=15, cookies=cookies_to_use_for_file, allow_redirects=True) as head_response: with requests.head(file_url, headers=file_download_headers, timeout=15, cookies=cookies_to_use_for_file, allow_redirects=True, proxies=self.proxies) as head_response:
head_response.raise_for_status() head_response.raise_for_status()
content_length = head_response.headers.get('Content-Length') content_length = head_response.headers.get('Content-Length')
if content_length: if content_length:
@@ -672,7 +673,7 @@ class PostProcessorWorker:
current_url_to_try = file_url current_url_to_try = file_url
response = requests.get(current_url_to_try, headers=file_download_headers, timeout=(30, 300), stream=True, cookies=cookies_to_use_for_file) response = requests.get(current_url_to_try, headers=file_download_headers, timeout=(30, 300), stream=True, cookies=cookies_to_use_for_file, proxies=self.proxies)
if response.status_code == 403 and ('kemono.' in current_url_to_try or 'coomer.' in current_url_to_try): if response.status_code == 403 and ('kemono.' in current_url_to_try or 'coomer.' in current_url_to_try):
self.logger(f" ⚠️ Got 403 Forbidden for '{api_original_filename}'. Attempting subdomain rotation...") self.logger(f" ⚠️ Got 403 Forbidden for '{api_original_filename}'. Attempting subdomain rotation...")
@@ -681,8 +682,7 @@ class PostProcessorWorker:
self.logger(f" Retrying with new URL: {new_url}") self.logger(f" Retrying with new URL: {new_url}")
file_url = new_url file_url = new_url
response.close() # Close the old response response.close() # Close the old response
response = requests.get(new_url, headers=file_download_headers, timeout=(30, 300), stream=True, cookies=cookies_to_use_for_file) response = requests.get(new_url, headers=file_download_headers, timeout=(30, 300), stream=True, cookies=cookies_to_use_for_file, proxies=self.proxies)
response.raise_for_status() response.raise_for_status()
# --- REVISED AND MOVED SIZE CHECK LOGIC --- # --- REVISED AND MOVED SIZE CHECK LOGIC ---
@@ -1104,8 +1104,8 @@ class PostProcessorWorker:
'Referer': creator_page_url, 'Referer': creator_page_url,
'Accept': 'text/css' 'Accept': 'text/css'
} }
cookies = prepare_cookies_for_request(self.use_cookie, self.cookie_text, self.selected_cookie_file, self.app_base_dir, self.logger, target_domain=api_domain) cookies = prepare_cookies_for_request(self.use_cookie, self.cookie_text, self.selected_cookie_file, self.app_base_dir, self.logger, target_domain=api_domain)
full_post_data = fetch_single_post_data(api_domain, self.service, self.user_id, post_id, headers, self.logger, cookies_dict=cookies) full_post_data = fetch_single_post_data(api_domain, self.service, self.user_id, post_id, headers, self.logger, cookies_dict=cookies, proxies=self.proxies)
if full_post_data: if full_post_data:
self.logger(" ✅ Full post data fetched successfully.") self.logger(" ✅ Full post data fetched successfully.")
self.post = full_post_data self.post = full_post_data
@@ -1306,13 +1306,17 @@ class PostProcessorWorker:
if not any(d in api_domain_for_comments.lower() for d in ['kemono.su', 'kemono.party', 'kemono.cr', 'coomer.su', 'coomer.party', 'coomer.st']): if not any(d in api_domain_for_comments.lower() for d in ['kemono.su', 'kemono.party', 'kemono.cr', 'coomer.su', 'coomer.party', 'coomer.st']):
self.logger(f"⚠️ Unrecognized domain '{api_domain_for_comments}' for comment API. Defaulting based on service.") self.logger(f"⚠️ Unrecognized domain '{api_domain_for_comments}' for comment API. Defaulting based on service.")
api_domain_for_comments = "kemono.cr" if "kemono" in self.service.lower() else "coomer.st" api_domain_for_comments = "kemono.cr" if "kemono" in self.service.lower() else "coomer.st"
# Fetch comments (Indented correctly now)
comments_data = fetch_post_comments( comments_data = fetch_post_comments(
api_domain_for_comments, self.service, self.user_id, post_id, api_domain_for_comments, self.service, self.user_id, post_id,
headers, self.logger, self.cancellation_event, self.pause_event, headers, self.logger, self.cancellation_event, self.pause_event,
cookies_dict=prepare_cookies_for_request( cookies_dict=prepare_cookies_for_request(
self.use_cookie, self.cookie_text, self.selected_cookie_file, self.app_base_dir, self.logger self.use_cookie, self.cookie_text, self.selected_cookie_file, self.app_base_dir, self.logger
) ),
proxies=self.proxies
) )
if comments_data: if comments_data:
self.logger(f" Fetched {len(comments_data)} comments for post {post_id}.") self.logger(f" Fetched {len(comments_data)} comments for post {post_id}.")
for comment_item_idx, comment_item in enumerate(comments_data): for comment_item_idx, comment_item in enumerate(comments_data):
@@ -1339,8 +1343,8 @@ class PostProcessorWorker:
except RuntimeError as e_fetch_comment: except RuntimeError as e_fetch_comment:
self.logger(f" ⚠️ Error fetching or processing comments for post {post_id}: {e_fetch_comment}") self.logger(f" ⚠️ Error fetching or processing comments for post {post_id}: {e_fetch_comment}")
except Exception as e_generic_comment: except Exception as e_generic_comment:
self.logger(f" ❌ Unexpected error during comment processing for post {post_id}: {e_generic_comment}\n{traceback.format_exc(limit=2)}") self.logger(f" ❌ Unexpected error during comment processing for post {post_id}: {e_generic_comment}\n{traceback.format_exc(limit=2)}")
self.logger(f" [Char Scope: Comments] Phase 2 Result: post_is_candidate_by_comment_char_match = {post_is_candidate_by_comment_char_match}")
else: else:
self.logger(f" [Char Scope: Comments] Phase 2: Skipped comment check for post ID '{post_id}' because a file match already made it a candidate.") self.logger(f" [Char Scope: Comments] Phase 2: Skipped comment check for post ID '{post_id}' because a file match already made it a candidate.")
@@ -2327,9 +2331,10 @@ class DownloadThread(QThread):
manga_custom_filename_format="{published} {title}", manga_custom_filename_format="{published} {title}",
manga_custom_date_format="YYYY-MM-DD" , manga_custom_date_format="YYYY-MM-DD" ,
sfp_threshold=None, sfp_threshold=None,
creator_name_cache=None creator_name_cache=None,
proxies=None
): ):
super().__init__() super().__init__()
self.api_url_input = api_url_input self.api_url_input = api_url_input
self.output_dir = output_dir self.output_dir = output_dir
@@ -2404,6 +2409,7 @@ class DownloadThread(QThread):
self.domain_override = domain_override self.domain_override = domain_override
self.sfp_threshold = sfp_threshold self.sfp_threshold = sfp_threshold
self.creator_name_cache = creator_name_cache self.creator_name_cache = creator_name_cache
self.proxies = proxies
if self.compress_images and Image is None: if self.compress_images and Image is None:
self.logger("⚠️ Image compression disabled: Pillow library not found (DownloadThread).") self.logger("⚠️ Image compression disabled: Pillow library not found (DownloadThread).")
@@ -2437,6 +2443,7 @@ class DownloadThread(QThread):
self.logger(" Starting post fetch (single-threaded download process)...") self.logger(" Starting post fetch (single-threaded download process)...")
# --- FIX: Removed duplicate proxies argument here ---
post_generator = download_from_api( post_generator = download_from_api(
self.api_url_input, self.api_url_input,
logger=self.logger, logger=self.logger,
@@ -2451,7 +2458,8 @@ class DownloadThread(QThread):
app_base_dir=self.app_base_dir, app_base_dir=self.app_base_dir,
manga_filename_style_for_sort_check=self.manga_filename_style if self.manga_mode_active else None, manga_filename_style_for_sort_check=self.manga_filename_style if self.manga_mode_active else None,
processed_post_ids=self.processed_post_ids_set, processed_post_ids=self.processed_post_ids_set,
fetch_all_first=self.fetch_first fetch_all_first=self.fetch_first,
proxies=self.proxies
) )
for posts_batch_data in post_generator: for posts_batch_data in post_generator:
@@ -2464,6 +2472,7 @@ class DownloadThread(QThread):
was_process_cancelled = True was_process_cancelled = True
break break
# --- FIX: Ensure 'proxies' is in this dictionary ---
worker_args = { worker_args = {
'post_data': individual_post_data, 'post_data': individual_post_data,
'emitter': worker_signals_obj, 'emitter': worker_signals_obj,
@@ -2532,7 +2541,8 @@ class DownloadThread(QThread):
'archive_only_mode': self.archive_only_mode, 'archive_only_mode': self.archive_only_mode,
'manga_custom_filename_format': self.manga_custom_filename_format, 'manga_custom_filename_format': self.manga_custom_filename_format,
'manga_custom_date_format': self.manga_custom_date_format, 'manga_custom_date_format': self.manga_custom_date_format,
'sfp_threshold': self.sfp_threshold 'sfp_threshold': self.sfp_threshold,
'proxies': self.proxies
} }
post_processing_worker = PostProcessorWorker(**worker_args) post_processing_worker = PostProcessorWorker(**worker_args)

View File

@@ -21,8 +21,7 @@ class DeviantArtDownloadThread(QThread):
self.pause_event = pause_event self.pause_event = pause_event
self.cancellation_event = cancellation_event self.cancellation_event = cancellation_event
# --- PASS LOGGER TO CLIENT --- # Pass logger to client so we see "Rate Limit" messages in the UI
# This ensures client logs go to the UI, not just the black console window
self.client = DeviantArtClient(logger_func=self.progress_signal.emit) self.client = DeviantArtClient(logger_func=self.progress_signal.emit)
self.parent_app = parent self.parent_app = parent
@@ -30,12 +29,13 @@ class DeviantArtDownloadThread(QThread):
self.skip_count = 0 self.skip_count = 0
# --- THREAD SETTINGS --- # --- THREAD SETTINGS ---
self.max_threads = 10 # STRICTLY 1 THREAD (Sequential) to match 1.py and avoid Rate Limits
self.max_threads = 1
def run(self): def run(self):
self.progress_signal.emit("=" * 40) self.progress_signal.emit("=" * 40)
self.progress_signal.emit(f"🚀 Starting DeviantArt download for: {self.url}") self.progress_signal.emit(f"🚀 Starting DeviantArt download for: {self.url}")
self.progress_signal.emit(f" Using {self.max_threads} parallel threads.") self.progress_signal.emit(f" Mode: Sequential (1 thread) to prevent 429 errors.")
try: try:
if not self.client.authenticate(): if not self.client.authenticate():
@@ -108,8 +108,10 @@ class DeviantArtDownloadThread(QThread):
future = executor.submit(self._process_deviation_task, deviation, base_folder) future = executor.submit(self._process_deviation_task, deviation, base_folder)
futures.append(future) futures.append(future)
# Wait for this batch to finish before getting the next page
wait(futures) wait(futures)
# Match 1.py: Sleep 1s between pages to be nice to API
time.sleep(1) time.sleep(1)
def _process_deviation_task(self, deviation, base_folder): def _process_deviation_task(self, deviation, base_folder):
@@ -119,6 +121,7 @@ class DeviantArtDownloadThread(QThread):
title = deviation.get('title', 'Unknown') title = deviation.get('title', 'Unknown')
try: try:
# This handles the fallback logic internally
content = self.client.get_deviation_content(dev_id) content = self.client.get_deviation_content(dev_id)
if content: if content:
self._download_file(content['src'], deviation, override_dir=base_folder) self._download_file(content['src'], deviation, override_dir=base_folder)
@@ -152,6 +155,7 @@ class DeviantArtDownloadThread(QThread):
final_filename = f"{safe_title}{ext}" final_filename = f"{safe_title}{ext}"
# Naming logic
if self.parent_app and self.parent_app.manga_mode_checkbox.isChecked(): if self.parent_app and self.parent_app.manga_mode_checkbox.isChecked():
try: try:
creator_name = metadata.get('author', {}).get('username', 'Unknown') creator_name = metadata.get('author', {}).get('username', 'Unknown')

View File

@@ -5,10 +5,11 @@ import sys
# --- PyQt5 Imports --- # --- PyQt5 Imports ---
from PyQt5.QtCore import Qt, QStandardPaths, QTimer from PyQt5.QtCore import Qt, QStandardPaths, QTimer
from PyQt5.QtGui import QIntValidator # <--- NEW: Added for Port validation
from PyQt5.QtWidgets import ( from PyQt5.QtWidgets import (
QApplication, QDialog, QHBoxLayout, QLabel, QPushButton, QVBoxLayout, QApplication, QDialog, QHBoxLayout, QLabel, QPushButton, QVBoxLayout,
QGroupBox, QComboBox, QMessageBox, QGridLayout, QCheckBox, QLineEdit, QGroupBox, QComboBox, QMessageBox, QGridLayout, QCheckBox, QLineEdit,
QTabWidget, QWidget, QFileDialog # Added QFileDialog QTabWidget, QWidget, QFileDialog
) )
# --- Local Application Imports --- # --- Local Application Imports ---
from ...i18n.translator import get_translation from ...i18n.translator import get_translation
@@ -21,7 +22,9 @@ from ...config.constants import (
RESOLUTION_KEY, UI_SCALE_KEY, SAVE_CREATOR_JSON_KEY, RESOLUTION_KEY, UI_SCALE_KEY, SAVE_CREATOR_JSON_KEY,
DATE_PREFIX_FORMAT_KEY, DATE_PREFIX_FORMAT_KEY,
COOKIE_TEXT_KEY, USE_COOKIE_KEY, COOKIE_TEXT_KEY, USE_COOKIE_KEY,
FETCH_FIRST_KEY, DISCORD_TOKEN_KEY, POST_DOWNLOAD_ACTION_KEY FETCH_FIRST_KEY, DISCORD_TOKEN_KEY, POST_DOWNLOAD_ACTION_KEY,
PROXY_ENABLED_KEY, PROXY_HOST_KEY, PROXY_PORT_KEY,
PROXY_USERNAME_KEY, PROXY_PASSWORD_KEY
) )
from ...services.updater import UpdateChecker, UpdateDownloader from ...services.updater import UpdateChecker, UpdateDownloader
@@ -118,16 +121,15 @@ class FutureSettingsDialog(QDialog):
super().__init__(parent) super().__init__(parent)
self.parent_app = parent_app_ref self.parent_app = parent_app_ref
self.setModal(True) self.setModal(True)
self.update_downloader_thread = None # To keep a reference self.update_downloader_thread = None
app_icon = get_app_icon_object() app_icon = get_app_icon_object()
if app_icon and not app_icon.isNull(): if app_icon and not app_icon.isNull():
self.setWindowIcon(app_icon) self.setWindowIcon(app_icon)
screen_height = QApplication.primaryScreen().availableGeometry().height() if QApplication.primaryScreen() else 800 screen_height = QApplication.primaryScreen().availableGeometry().height() if QApplication.primaryScreen() else 800
# Use a more balanced aspect ratio
scale_factor = screen_height / 1000.0 scale_factor = screen_height / 1000.0
base_min_w, base_min_h = 480, 420 # Wider, less tall base_min_w, base_min_h = 550, 450 # <--- TWEAK: Slightly increased width for better layout
scaled_min_w = int(base_min_w * scale_factor) scaled_min_w = int(base_min_w * scale_factor)
scaled_min_h = int(base_min_h * scale_factor) scaled_min_h = int(base_min_h * scale_factor)
self.setMinimumSize(scaled_min_w, scaled_min_h) self.setMinimumSize(scaled_min_w, scaled_min_h)
@@ -135,6 +137,9 @@ class FutureSettingsDialog(QDialog):
self._init_ui() self._init_ui()
self._retranslate_ui() self._retranslate_ui()
self._apply_theme() self._apply_theme()
# <--- NEW: Load proxy settings on init
self._load_proxy_settings()
def _init_ui(self): def _init_ui(self):
"""Initializes all UI components and layouts for the dialog.""" """Initializes all UI components and layouts for the dialog."""
@@ -147,14 +152,16 @@ class FutureSettingsDialog(QDialog):
# --- Create Tabs --- # --- Create Tabs ---
self.display_tab = QWidget() self.display_tab = QWidget()
self.downloads_tab = QWidget() self.downloads_tab = QWidget()
self.network_tab = QWidget() # <--- NEW: Network Tab
self.updates_tab = QWidget() self.updates_tab = QWidget()
# Add tabs to the widget # Add tabs to the widget
self.tab_widget.addTab(self.display_tab, "Display") self.tab_widget.addTab(self.display_tab, "Display")
self.tab_widget.addTab(self.downloads_tab, "Downloads") self.tab_widget.addTab(self.downloads_tab, "Downloads")
self.tab_widget.addTab(self.network_tab, "Proxy/Network") # <--- NEW
self.tab_widget.addTab(self.updates_tab, "Updates") self.tab_widget.addTab(self.updates_tab, "Updates")
# --- Populate Display Tab --- # [Display Tab Code (Unchanged) ...]
display_tab_layout = QVBoxLayout(self.display_tab) display_tab_layout = QVBoxLayout(self.display_tab)
self.display_group_box = QGroupBox() self.display_group_box = QGroupBox()
display_layout = QGridLayout(self.display_group_box) display_layout = QGridLayout(self.display_group_box)
@@ -184,9 +191,9 @@ class FutureSettingsDialog(QDialog):
display_layout.addWidget(self.resolution_combo_box, 3, 1) display_layout.addWidget(self.resolution_combo_box, 3, 1)
display_tab_layout.addWidget(self.display_group_box) display_tab_layout.addWidget(self.display_group_box)
display_tab_layout.addStretch(1) # Push content to the top display_tab_layout.addStretch(1)
# --- Populate Downloads Tab --- # [Downloads Tab Code (Unchanged) ...]
downloads_tab_layout = QVBoxLayout(self.downloads_tab) downloads_tab_layout = QVBoxLayout(self.downloads_tab)
self.download_settings_group_box = QGroupBox() self.download_settings_group_box = QGroupBox()
download_settings_layout = QGridLayout(self.download_settings_group_box) download_settings_layout = QGridLayout(self.download_settings_group_box)
@@ -217,7 +224,6 @@ class FutureSettingsDialog(QDialog):
self.fetch_first_checkbox.stateChanged.connect(self._fetch_first_setting_changed) self.fetch_first_checkbox.stateChanged.connect(self._fetch_first_setting_changed)
download_settings_layout.addWidget(self.fetch_first_checkbox, 4, 0, 1, 2) download_settings_layout.addWidget(self.fetch_first_checkbox, 4, 0, 1, 2)
# --- START: Add new Load/Save buttons ---
settings_file_layout = QHBoxLayout() settings_file_layout = QHBoxLayout()
self.load_settings_button = QPushButton() self.load_settings_button = QPushButton()
self.save_settings_button = QPushButton() self.save_settings_button = QPushButton()
@@ -225,18 +231,63 @@ class FutureSettingsDialog(QDialog):
settings_file_layout.addWidget(self.save_settings_button) settings_file_layout.addWidget(self.save_settings_button)
settings_file_layout.addStretch(1) settings_file_layout.addStretch(1)
# Add this new layout to the grid download_settings_layout.addLayout(settings_file_layout, 5, 0, 1, 2)
download_settings_layout.addLayout(settings_file_layout, 5, 0, 1, 2) # Row 5, span 2 cols
# Connect signals
self.load_settings_button.clicked.connect(self._handle_load_settings) self.load_settings_button.clicked.connect(self._handle_load_settings)
self.save_settings_button.clicked.connect(self._handle_save_settings) self.save_settings_button.clicked.connect(self._handle_save_settings)
# --- END: Add new Load/Save buttons ---
downloads_tab_layout.addWidget(self.download_settings_group_box) downloads_tab_layout.addWidget(self.download_settings_group_box)
downloads_tab_layout.addStretch(1) # Push content to the top downloads_tab_layout.addStretch(1)
# --- Populate Updates Tab --- # --- START: Network Tab (NEW) ---
network_tab_layout = QVBoxLayout(self.network_tab)
self.proxy_group_box = QGroupBox()
proxy_layout = QGridLayout(self.proxy_group_box)
# Enable Checkbox
self.proxy_enabled_checkbox = QCheckBox()
self.proxy_enabled_checkbox.stateChanged.connect(self._proxy_setting_changed)
proxy_layout.addWidget(self.proxy_enabled_checkbox, 0, 0, 1, 2)
# Host / IP
self.proxy_host_label = QLabel()
self.proxy_host_input = QLineEdit()
self.proxy_host_input.setPlaceholderText("127.0.0.1")
self.proxy_host_input.editingFinished.connect(self._proxy_setting_changed)
proxy_layout.addWidget(self.proxy_host_label, 1, 0)
proxy_layout.addWidget(self.proxy_host_input, 1, 1)
# Port
self.proxy_port_label = QLabel()
self.proxy_port_input = QLineEdit()
self.proxy_port_input.setPlaceholderText("8080")
self.proxy_port_input.setValidator(QIntValidator(1, 65535, self)) # Only numbers
self.proxy_port_input.editingFinished.connect(self._proxy_setting_changed)
proxy_layout.addWidget(self.proxy_port_label, 2, 0)
proxy_layout.addWidget(self.proxy_port_input, 2, 1)
# Username
self.proxy_user_label = QLabel()
self.proxy_user_input = QLineEdit()
self.proxy_user_input.setPlaceholderText("(Optional)")
self.proxy_user_input.editingFinished.connect(self._proxy_setting_changed)
proxy_layout.addWidget(self.proxy_user_label, 3, 0)
proxy_layout.addWidget(self.proxy_user_input, 3, 1)
# Password
self.proxy_pass_label = QLabel()
self.proxy_pass_input = QLineEdit()
self.proxy_pass_input.setPlaceholderText("(Optional)")
self.proxy_pass_input.setEchoMode(QLineEdit.Password) # Mask input
self.proxy_pass_input.editingFinished.connect(self._proxy_setting_changed)
proxy_layout.addWidget(self.proxy_pass_label, 4, 0)
proxy_layout.addWidget(self.proxy_pass_input, 4, 1)
network_tab_layout.addWidget(self.proxy_group_box)
network_tab_layout.addStretch(1)
# --- END: Network Tab (NEW) ---
# [Updates Tab Code (Unchanged) ...]
updates_tab_layout = QVBoxLayout(self.updates_tab) updates_tab_layout = QVBoxLayout(self.updates_tab)
self.update_group_box = QGroupBox() self.update_group_box = QGroupBox()
update_layout = QGridLayout(self.update_group_box) update_layout = QGridLayout(self.update_group_box)
@@ -249,7 +300,7 @@ class FutureSettingsDialog(QDialog):
update_layout.addWidget(self.check_update_button, 1, 0, 1, 2) update_layout.addWidget(self.check_update_button, 1, 0, 1, 2)
updates_tab_layout.addWidget(self.update_group_box) updates_tab_layout.addWidget(self.update_group_box)
updates_tab_layout.addStretch(1) # Push content to the top updates_tab_layout.addStretch(1)
# --- OK Button (outside tabs) --- # --- OK Button (outside tabs) ---
button_layout = QHBoxLayout() button_layout = QHBoxLayout()
@@ -266,16 +317,17 @@ class FutureSettingsDialog(QDialog):
# --- Tab Titles --- # --- Tab Titles ---
self.tab_widget.setTabText(0, self._tr("settings_tab_display", "Display")) self.tab_widget.setTabText(0, self._tr("settings_tab_display", "Display"))
self.tab_widget.setTabText(1, self._tr("settings_tab_downloads", "Downloads")) self.tab_widget.setTabText(1, self._tr("settings_tab_downloads", "Downloads"))
self.tab_widget.setTabText(2, self._tr("settings_tab_updates", "Updates")) self.tab_widget.setTabText(2, self._tr("settings_tab_network", "Proxy/Network")) # <--- NEW
self.tab_widget.setTabText(3, self._tr("settings_tab_updates", "Updates"))
# --- Display Tab --- # [Display Tab (Unchanged) ...]
self.display_group_box.setTitle(self._tr("display_settings_group_title", "Display Settings")) self.display_group_box.setTitle(self._tr("display_settings_group_title", "Display Settings"))
self.theme_label.setText(self._tr("theme_label", "Theme:")) self.theme_label.setText(self._tr("theme_label", "Theme:"))
self.ui_scale_label.setText(self._tr("ui_scale_label", "UI Scale:")) self.ui_scale_label.setText(self._tr("ui_scale_label", "UI Scale:"))
self.language_label.setText(self._tr("language_label", "Language:")) self.language_label.setText(self._tr("language_label", "Language:"))
self.window_size_label.setText(self._tr("window_size_label", "Window Size:")) self.window_size_label.setText(self._tr("window_size_label", "Window Size:"))
# --- Downloads Tab --- # [Downloads Tab (Unchanged) ...]
self.download_settings_group_box.setTitle(self._tr("download_settings_group_title", "Download Settings")) self.download_settings_group_box.setTitle(self._tr("download_settings_group_title", "Download Settings"))
self.default_path_label.setText(self._tr("default_path_label", "Default Path:")) self.default_path_label.setText(self._tr("default_path_label", "Default Path:"))
self.date_prefix_format_label.setText(self._tr("date_prefix_format_label", "Post Subfolder Format:")) self.date_prefix_format_label.setText(self._tr("date_prefix_format_label", "Post Subfolder Format:"))
@@ -294,32 +346,93 @@ class FutureSettingsDialog(QDialog):
self.fetch_first_checkbox.setToolTip(self._tr("fetch_first_tooltip", "If checked, the downloader will find all posts from a creator first before starting any downloads.\nThis can be slower to start but provides a more accurate progress bar.")) self.fetch_first_checkbox.setToolTip(self._tr("fetch_first_tooltip", "If checked, the downloader will find all posts from a creator first before starting any downloads.\nThis can be slower to start but provides a more accurate progress bar."))
self.save_path_button.setText(self._tr("settings_save_all_button", "Save Path + Cookie + Token")) self.save_path_button.setText(self._tr("settings_save_all_button", "Save Path + Cookie + Token"))
self.save_path_button.setToolTip(self._tr("settings_save_all_tooltip", "Save the current 'Download Location', Cookie, and Discord Token settings for future sessions.")) self.save_path_button.setToolTip(self._tr("settings_save_all_tooltip", "Save the current 'Download Location', Cookie, and Discord Token settings for future sessions."))
# --- START: Add new button text ---
self.load_settings_button.setText(self._tr("load_settings_button", "Load Settings...")) self.load_settings_button.setText(self._tr("load_settings_button", "Load Settings..."))
self.load_settings_button.setToolTip(self._tr("load_settings_tooltip", "Load all download settings from a .json file.")) self.load_settings_button.setToolTip(self._tr("load_settings_tooltip", "Load all download settings from a .json file."))
self.save_settings_button.setText(self._tr("save_settings_button", "Save Settings...")) self.save_settings_button.setText(self._tr("save_settings_button", "Save Settings..."))
self.save_settings_button.setToolTip(self._tr("save_settings_tooltip", "Save all current download settings to a .json file.")) self.save_settings_button.setToolTip(self._tr("save_settings_tooltip", "Save all current download settings to a .json file."))
# --- END: Add new button text ---
# --- Updates Tab --- # --- START: Network Tab (NEW) ---
self.proxy_group_box.setTitle(self._tr("proxy_settings_group_title", "Proxy Configuration"))
self.proxy_enabled_checkbox.setText(self._tr("proxy_enabled_label", "Enable Proxy"))
self.proxy_host_label.setText(self._tr("proxy_host_label", "Host / IP:"))
self.proxy_port_label.setText(self._tr("proxy_port_label", "Port:"))
self.proxy_user_label.setText(self._tr("proxy_user_label", "Username (Optional):"))
self.proxy_pass_label.setText(self._tr("proxy_pass_label", "Password (Optional):"))
# --- END: Network Tab (NEW) ---
# [Updates Tab (Unchanged) ...]
self.update_group_box.setTitle(self._tr("update_group_title", "Application Updates")) self.update_group_box.setTitle(self._tr("update_group_title", "Application Updates"))
current_version = self.parent_app.windowTitle().split(' v')[-1] current_version = self.parent_app.windowTitle().split(' v')[-1]
self.version_label.setText(self._tr("current_version_label", f"Current Version: v{current_version}")) self.version_label.setText(self._tr("current_version_label", f"Current Version: v{current_version}"))
self.update_status_label.setText(self._tr("update_status_ready", "Ready to check.")) self.update_status_label.setText(self._tr("update_status_ready", "Ready to check."))
self.check_update_button.setText(self._tr("check_for_updates_button", "Check for Updates")) self.check_update_button.setText(self._tr("check_for_updates_button", "Check for Updates"))
# --- General ---
self._update_theme_toggle_button_text() self._update_theme_toggle_button_text()
self.ok_button.setText(self._tr("ok_button", "OK")) self.ok_button.setText(self._tr("ok_button", "OK"))
# --- Load Data ---
self._populate_display_combo_boxes() self._populate_display_combo_boxes()
self._populate_language_combo_box() self._populate_language_combo_box()
self._populate_post_download_action_combo() self._populate_post_download_action_combo()
self._load_date_prefix_format() self._load_date_prefix_format()
self._load_checkbox_states() self._load_checkbox_states()
# --- START: New Proxy Logic ---
def _load_proxy_settings(self):
"""Loads proxy settings from QSettings into the UI."""
self.proxy_enabled_checkbox.blockSignals(True)
self.proxy_host_input.blockSignals(True)
self.proxy_port_input.blockSignals(True)
self.proxy_user_input.blockSignals(True)
self.proxy_pass_input.blockSignals(True)
enabled = self.parent_app.settings.value(PROXY_ENABLED_KEY, False, type=bool)
host = self.parent_app.settings.value(PROXY_HOST_KEY, "", type=str)
port = self.parent_app.settings.value(PROXY_PORT_KEY, "", type=str)
user = self.parent_app.settings.value(PROXY_USERNAME_KEY, "", type=str)
password = self.parent_app.settings.value(PROXY_PASSWORD_KEY, "", type=str)
self.proxy_enabled_checkbox.setChecked(enabled)
self.proxy_host_input.setText(host)
self.proxy_port_input.setText(port)
self.proxy_user_input.setText(user)
self.proxy_pass_input.setText(password)
self._update_proxy_fields_state(enabled)
self.proxy_enabled_checkbox.blockSignals(False)
self.proxy_host_input.blockSignals(False)
self.proxy_port_input.blockSignals(False)
self.proxy_user_input.blockSignals(False)
self.proxy_pass_input.blockSignals(False)
def _proxy_setting_changed(self):
"""Saves the current proxy UI state to QSettings."""
enabled = self.proxy_enabled_checkbox.isChecked()
host = self.proxy_host_input.text().strip()
port = self.proxy_port_input.text().strip()
user = self.proxy_user_input.text().strip()
password = self.proxy_pass_input.text().strip()
self.parent_app.settings.setValue(PROXY_ENABLED_KEY, enabled)
self.parent_app.settings.setValue(PROXY_HOST_KEY, host)
self.parent_app.settings.setValue(PROXY_PORT_KEY, port)
self.parent_app.settings.setValue(PROXY_USERNAME_KEY, user)
self.parent_app.settings.setValue(PROXY_PASSWORD_KEY, password)
self.parent_app.settings.sync()
self._update_proxy_fields_state(enabled)
# Optional: Notify main app that network settings changed if needed
# self.parent_app.reload_proxy_settings()
def _update_proxy_fields_state(self, enabled):
"""Enables or disables input fields based on the checkbox."""
self.proxy_host_input.setEnabled(enabled)
self.proxy_port_input.setEnabled(enabled)
self.proxy_user_input.setEnabled(enabled)
self.proxy_pass_input.setEnabled(enabled)
# --- END: New Proxy Logic ---
def _check_for_updates(self): def _check_for_updates(self):
self.check_update_button.setEnabled(False) self.check_update_button.setEnabled(False)
self.update_status_label.setText(self._tr("update_status_checking", "Checking...")) self.update_status_label.setText(self._tr("update_status_checking", "Checking..."))

View File

@@ -1,6 +1,7 @@
import sys import sys
import os import os
import time import time
import glob
import queue import queue
import random import random
import traceback import traceback
@@ -164,7 +165,7 @@ class DownloaderApp (QWidget ):
self.is_finishing = False self.is_finishing = False
self.finish_lock = threading.Lock() self.finish_lock = threading.Lock()
self.add_info_in_pdf_setting = False self.add_info_in_pdf_setting = False
saved_res = self.settings.value(RESOLUTION_KEY, "Auto") saved_res = self.settings.value(RESOLUTION_KEY, "Auto")
if saved_res != "Auto": if saved_res != "Auto":
try: try:
@@ -187,6 +188,11 @@ class DownloaderApp (QWidget ):
self.user_data_path = user_data_path self.user_data_path = user_data_path
self.jobs_dir = os.path.join(self.user_data_path, "jobs")
os.makedirs(self.jobs_dir, exist_ok=True)
self.is_running_job_queue = False
self.current_job_file = None
self.config_file = os.path.join(user_data_path, "Known.txt") self.config_file = os.path.join(user_data_path, "Known.txt")
self.session_file_path = os.path.join(user_data_path, "session.json") self.session_file_path = os.path.join(user_data_path, "session.json")
self.persistent_history_file = os.path.join(user_data_path, "download_history.json") self.persistent_history_file = os.path.join(user_data_path, "download_history.json")
@@ -357,6 +363,183 @@ class DownloaderApp (QWidget ):
self._check_for_interrupted_session() self._check_for_interrupted_session()
self._cleanup_after_update() self._cleanup_after_update()
def add_current_settings_to_queue(self):
"""Saves the current UI settings as a JSON job file with creator-specific paths."""
# --- Helper: Append Name to Path safely ---
def get_creator_specific_path(base_dir, folder_name):
if not folder_name:
return base_dir
safe_name = clean_folder_name(folder_name)
# Avoid double pathing (e.g. if base is .../Artist and we append /Artist again)
if base_dir.replace('\\', '/').rstrip('/').endswith(safe_name):
return base_dir
return os.path.join(base_dir, safe_name)
# ------------------------------------------
# --- SCENARIO 1: Items from Creator Selection (Popup) ---
if self.favorite_download_queue:
count = 0
base_settings = self._get_current_ui_settings_as_dict()
items_to_process = list(self.favorite_download_queue)
for item in items_to_process:
real_url = item.get('url')
name = item.get('name', 'Unknown')
if not real_url: continue
job_settings = base_settings.copy()
job_settings['api_url'] = real_url
# Use the name provided by the selection popup
job_settings['output_dir'] = get_creator_specific_path(job_settings['output_dir'], name)
if self._save_single_job_file(job_settings, name_hint=name):
count += 1
if count > 0:
self.log_signal.emit(f"✅ Added {count} jobs to queue from selection.")
self.link_input.clear()
self.favorite_download_queue.clear()
QMessageBox.information(self, "Queue", f"{count} jobs successfully added to queue!")
else:
QMessageBox.warning(self, "Queue Error", "Failed to add selected items to queue.")
return
# --- SCENARIO 2: Manual URL Entry ---
url = self.link_input.text().strip()
if not url:
QMessageBox.warning(self, "Input Error", "Cannot add to queue: URL is empty.")
return
settings = self._get_current_ui_settings_as_dict()
settings['api_url'] = url
# Attempt to resolve name from URL + Cache (creators.json)
service, user_id, post_id = extract_post_info(url)
name_hint = "Job"
if service and user_id:
# Try to find name in your local creators cache
cache_key = (service.lower(), str(user_id))
cached_name = self.creator_name_cache.get(cache_key)
if cached_name:
# CASE A: Creator Found -> Use Creator Name
name_hint = cached_name
settings['output_dir'] = get_creator_specific_path(settings['output_dir'], cached_name)
else:
# CASE B: Creator NOT Found -> Use Post ID or User ID
# If it's a single post link, 'post_id' will have a value.
# If it's a profile link, 'post_id' is None, so we use 'user_id'.
if post_id:
folder_name = str(post_id)
else:
folder_name = str(user_id)
name_hint = folder_name
settings['output_dir'] = get_creator_specific_path(settings['output_dir'], folder_name)
if self._save_single_job_file(settings, name_hint=name_hint):
self.log_signal.emit(f"✅ Job added to queue: {url}")
self.link_input.clear()
QMessageBox.information(self, "Queue", "Job successfully added to queue!")
def _save_single_job_file(self, settings_dict, name_hint="job"):
"""Helper to write a single JSON job file to the jobs directory."""
import uuid
timestamp = int(time.time())
unique_id = uuid.uuid4().hex[:6]
# Clean the name hint to be safe for filenames
safe_name = "".join(c for c in name_hint if c.isalnum() or c in (' ', '_', '-')).strip()
if not safe_name:
safe_name = "job"
filename = f"job_{timestamp}_{safe_name}_{unique_id}.json"
filepath = os.path.join(self.jobs_dir, filename)
try:
with open(filepath, 'w', encoding='utf-8') as f:
json.dump(settings_dict, f, indent=2)
return True
except Exception as e:
self.log_signal.emit(f"❌ Failed to save job file '{filename}': {e}")
return False
def execute_job_queue(self):
"""Starts the queue processing loop."""
job_files = sorted(glob.glob(os.path.join(self.jobs_dir, "job_*.json")))
if not job_files:
QMessageBox.information(self, "Queue Empty", "No job files found in appdata/jobs.")
return
# --- FIX: Clear error log at the start of the entire queue session ---
self.permanently_failed_files_for_dialog.clear()
self._update_error_button_count()
# -------------------------------------------------------------------
self.log_signal.emit("=" * 40)
self.log_signal.emit(f"🚀 Starting execution of {len(job_files)} queued jobs.")
self.is_running_job_queue = True
self.download_btn.setEnabled(False) # Disable button while running
self.add_queue_btn.setEnabled(False)
self._process_next_queued_job()
def _process_next_queued_job(self):
"""Loads the next job file and starts the download."""
if self.cancellation_event.is_set():
self.is_running_job_queue = False
self.log_signal.emit("🛑 Queue execution cancelled.")
self._update_button_states_and_connections()
return
job_files = sorted(glob.glob(os.path.join(self.jobs_dir, "job_*.json")))
if not job_files:
self.is_running_job_queue = False
self.current_job_file = None
self.log_signal.emit("🏁 All queued jobs finished!")
self.link_input.clear()
QMessageBox.information(self, "Queue Finished", "All queued jobs have been processed.")
self._update_button_states_and_connections()
return
next_job_path = job_files[0]
self.current_job_file = next_job_path
self.log_signal.emit(f"📂 Loading job: {os.path.basename(next_job_path)}")
try:
with open(next_job_path, 'r', encoding='utf-8') as f:
settings = json.load(f)
# --- Ensure Directory Exists ---
# The settings now contain the full path (e.g. E:/Kemono/ArtistName)
target_dir = settings.get('output_dir', '')
if target_dir:
try:
os.makedirs(target_dir, exist_ok=True)
except Exception as e:
self.log_signal.emit(f"⚠️ Warning: Could not pre-create directory '{target_dir}': {e}")
# -------------------------------
# Load settings into UI
self._load_ui_from_settings_dict(settings)
QCoreApplication.processEvents()
# Start download
self.start_download()
except Exception as e:
self.log_signal.emit(f"❌ Error loading/starting job '{next_job_path}': {e}")
failed_path = next_job_path + ".failed"
os.rename(next_job_path, failed_path)
self._process_next_queued_job()
def _run_discord_file_download_thread(self, session, server_id, channel_id, token, output_dir, message_limit=None): def _run_discord_file_download_thread(self, session, server_id, channel_id, token, output_dir, message_limit=None):
""" """
Runs in a background thread to fetch and download all files from a Discord channel. Runs in a background thread to fetch and download all files from a Discord channel.
@@ -660,7 +843,20 @@ class DownloaderApp (QWidget ):
settings['add_info_in_pdf'] = self.add_info_in_pdf_setting # Save to settings dict settings['add_info_in_pdf'] = self.add_info_in_pdf_setting # Save to settings dict
settings['keep_duplicates_mode'] = self.keep_duplicates_mode settings['keep_duplicates_mode'] = self.keep_duplicates_mode
settings['keep_duplicates_limit'] = self.keep_duplicates_limit settings['keep_duplicates_limit'] = self.keep_duplicates_limit
settings['proxy_enabled'] = self.settings.value(PROXY_ENABLED_KEY, False, type=bool)
settings['proxy_host'] = self.settings.value(PROXY_HOST_KEY, "", type=str)
settings['proxy_port'] = self.settings.value(PROXY_PORT_KEY, "", type=str)
settings['proxy_username'] = self.settings.value(PROXY_USERNAME_KEY, "", type=str)
settings['proxy_password'] = self.settings.value(PROXY_PASSWORD_KEY, "", type=str)
settings['proxies'] = None
if settings['proxy_enabled'] and settings['proxy_host'] and settings['proxy_port']:
proxy_str = f"http://{settings['proxy_host']}:{settings['proxy_port']}"
if settings['proxy_username'] and settings['proxy_password']:
proxy_str = f"http://{settings['proxy_username']}:{settings['proxy_password']}@{settings['proxy_host']}:{settings['proxy_port']}"
settings['proxies'] = {'http': proxy_str, 'https': proxy_str}
return settings return settings
@@ -769,6 +965,23 @@ class DownloaderApp (QWidget ):
is_download_active = self._is_download_active() is_download_active = self._is_download_active()
fetch_first_enabled = self.settings.value(FETCH_FIRST_KEY, False, type=bool) fetch_first_enabled = self.settings.value(FETCH_FIRST_KEY, False, type=bool)
url_text = self.link_input.text().strip()
# --- NEW: Check for Queue Command ---
is_queue_command = (url_text.lower() == "start queue")
# --- NEW: Handle 'Add to Queue' Button State ---
if hasattr(self, 'add_queue_btn'):
# Only enable if not downloading, URL is valid, not in queue mode, and not in specialized fetch states
should_enable_queue = (
not is_download_active and
url_text != "" and
not is_queue_command and
not self.is_ready_to_download_fetched and
not self.is_ready_to_download_batch_update
)
self.add_queue_btn.setEnabled(should_enable_queue)
print(f"--- DEBUG: Updating buttons (is_download_active={is_download_active}) ---") print(f"--- DEBUG: Updating buttons (is_download_active={is_download_active}) ---")
if self.is_ready_to_download_fetched: if self.is_ready_to_download_fetched:
@@ -852,7 +1065,12 @@ class DownloaderApp (QWidget ):
self.download_btn.setText(f"⬇️ Start Download ({num_posts} Posts)") self.download_btn.setText(f"⬇️ Start Download ({num_posts} Posts)")
self.download_btn.setEnabled(True) # Keep it enabled for the user to click self.download_btn.setEnabled(True) # Keep it enabled for the user to click
else: else:
self.download_btn.setText(self._tr("start_download_button_text", "⬇️ Start Download")) # Check if running queue to show specific text
if hasattr(self, 'is_running_job_queue') and self.is_running_job_queue:
self.download_btn.setText("🔄 Processing Queue...")
else:
self.download_btn.setText(self._tr("start_download_button_text", "⬇️ Start Download"))
self.download_btn.setEnabled(False) self.download_btn.setEnabled(False)
self.pause_btn.setText(self._tr("resume_download_button_text", "▶️ Resume Download") if self.is_paused else self._tr("pause_download_button_text", "⏸️ Pause Download")) self.pause_btn.setText(self._tr("resume_download_button_text", "▶️ Resume Download") if self.is_paused else self._tr("pause_download_button_text", "⏸️ Pause Download"))
@@ -865,22 +1083,32 @@ class DownloaderApp (QWidget ):
self.cancel_btn.clicked.connect(self.cancel_download_button_action) self.cancel_btn.clicked.connect(self.cancel_download_button_action)
else: else:
url_text = self.link_input.text().strip() # --- IDLE STATE ---
_, _, post_id = extract_post_info(url_text) if is_queue_command:
is_single_post = bool(post_id) # --- NEW: Queue Execution Mode ---
self.download_btn.setText("🚀 Execute Queue")
if fetch_first_enabled and not is_single_post: self.download_btn.setEnabled(True)
self.download_btn.setText("📄 Fetch Pages") # Ensure the method exists before connecting
if hasattr(self, 'execute_job_queue'):
self.download_btn.clicked.connect(self.execute_job_queue)
else: else:
self.download_btn.setText(self._tr("start_download_button_text", "⬇️ Start Download")) _, _, post_id = extract_post_info(url_text)
is_single_post = bool(post_id)
self.download_btn.setEnabled(True)
self.download_btn.clicked.connect(self.start_download) if fetch_first_enabled and not is_single_post and url_text:
self.download_btn.setText("📄 Fetch Pages")
else:
self.download_btn.setText(self._tr("start_download_button_text", "⬇️ Start Download"))
self.download_btn.setEnabled(True)
self.download_btn.clicked.connect(self.start_download)
self.pause_btn.setText(self._tr("pause_download_button_text", "⏸️ Pause Download")) self.pause_btn.setText(self._tr("pause_download_button_text", "⏸️ Pause Download"))
self.pause_btn.setEnabled(False) self.pause_btn.setEnabled(False)
self.cancel_btn.setText(self._tr("cancel_button_text", "❌ Cancel & Reset UI")) self.cancel_btn.setText(self._tr("cancel_button_text", "❌ Cancel & Reset UI"))
self.cancel_btn.setEnabled(False) self.cancel_btn.setEnabled(False)
def _run_fetch_only_thread(self, fetch_args): def _run_fetch_only_thread(self, fetch_args):
""" """
Runs in a background thread to ONLY fetch all posts without downloading. Runs in a background thread to ONLY fetch all posts without downloading.
@@ -3826,6 +4054,10 @@ class DownloaderApp (QWidget ):
self.downloaded_hash_counts.clear() self.downloaded_hash_counts.clear()
if not is_restore and not is_continuation: if not is_restore and not is_continuation:
if not self.is_running_job_queue:
self.permanently_failed_files_for_dialog.clear()
self.permanently_failed_files_for_dialog.clear() self.permanently_failed_files_for_dialog.clear()
self.retryable_failed_files_info.clear() self.retryable_failed_files_info.clear()
@@ -4408,6 +4640,14 @@ class DownloaderApp (QWidget ):
if should_use_multithreading_for_posts: if should_use_multithreading_for_posts:
log_messages.append(f" Number of Post Worker Threads: {effective_num_post_workers}") log_messages.append(f" Number of Post Worker Threads: {effective_num_post_workers}")
proxy_enabled_log = self.settings.value(PROXY_ENABLED_KEY, False, type=bool)
if proxy_enabled_log:
p_host = self.settings.value(PROXY_HOST_KEY, "")
p_port = self.settings.value(PROXY_PORT_KEY, "")
log_messages.append(f" Proxy: Enabled ({p_host}:{p_port})")
else:
log_messages.append(f" Proxy: Disabled")
if domain_override_command: if domain_override_command:
self.log_signal.emit(f" Domain Override Active: Will probe for the correct 'n*' subdomain on the '.{domain_override_command}' domain for each file.") self.log_signal.emit(f" Domain Override Active: Will probe for the correct 'n*' subdomain on the '.{domain_override_command}' domain for each file.")
@@ -4420,7 +4660,7 @@ class DownloaderApp (QWidget ):
self.set_ui_enabled(False) self.set_ui_enabled(False)
from src.config.constants import FOLDER_NAME_STOP_WORDS from src.config.constants import FOLDER_NAME_STOP_WORDS
current_proxies = self._get_current_ui_settings_as_dict().get('proxies')
args_template = { args_template = {
'api_url_input': api_url, 'api_url_input': api_url,
'download_root': effective_output_dir_for_run, 'download_root': effective_output_dir_for_run,
@@ -4496,7 +4736,8 @@ class DownloaderApp (QWidget ):
'fetch_first': fetch_first_enabled, 'fetch_first': fetch_first_enabled,
'sfp_threshold': download_commands.get('sfp_threshold'), 'sfp_threshold': download_commands.get('sfp_threshold'),
'handle_unknown_mode': handle_unknown_command, 'handle_unknown_mode': handle_unknown_command,
'add_info_in_pdf': self.add_info_in_pdf_setting, 'add_info_in_pdf': self.add_info_in_pdf_setting,
'proxies': current_proxies
} }
args_template['override_output_dir'] = override_output_dir args_template['override_output_dir'] = override_output_dir
@@ -4522,7 +4763,8 @@ class DownloaderApp (QWidget ):
'app_base_dir': app_base_dir_for_cookies, 'app_base_dir': app_base_dir_for_cookies,
'manga_filename_style_for_sort_check': self.manga_filename_style, 'manga_filename_style_for_sort_check': self.manga_filename_style,
'processed_post_ids': processed_post_ids_for_this_run, 'processed_post_ids': processed_post_ids_for_this_run,
'fetch_all_first': True 'fetch_all_first': True,
'proxies': self._get_current_ui_settings_as_dict().get('proxies')
} }
self.download_thread = threading.Thread(target=self._run_fetch_only_thread, args=(fetch_thread_args,), daemon=True) self.download_thread = threading.Thread(target=self._run_fetch_only_thread, args=(fetch_thread_args,), daemon=True)
@@ -4878,8 +5120,7 @@ class DownloaderApp (QWidget ):
ppw_expected_keys = list(PostProcessorWorker.__init__.__code__.co_varnames)[1:] ppw_expected_keys = list(PostProcessorWorker.__init__.__code__.co_varnames)[1:]
# 1. Define all LIVE RUNTIME arguments. current_proxies = self._get_current_ui_settings_as_dict().get('proxies')
# These are taken from the current app state and are the same for all workers.
live_runtime_args = { live_runtime_args = {
'emitter': self.worker_to_gui_queue, 'emitter': self.worker_to_gui_queue,
'creator_name_cache': self.creator_name_cache, 'creator_name_cache': self.creator_name_cache,
@@ -4909,7 +5150,8 @@ class DownloaderApp (QWidget ):
'use_cookie': self.use_cookie_checkbox.isChecked(), 'use_cookie': self.use_cookie_checkbox.isChecked(),
'cookie_text': self.cookie_text_input.text(), 'cookie_text': self.cookie_text_input.text(),
'selected_cookie_file': self.selected_cookie_filepath, 'selected_cookie_file': self.selected_cookie_filepath,
'add_info_in_pdf': self.add_info_in_pdf_setting, 'add_info_in_pdf': self.add_info_in_pdf_setting,
'proxies': current_proxies,
} }
# 2. Define DEFAULTS for all settings that *should* be in the profile. # 2. Define DEFAULTS for all settings that *should* be in the profile.
@@ -5145,6 +5387,19 @@ class DownloaderApp (QWidget ):
self._update_manga_filename_style_button_text() self._update_manga_filename_style_button_text()
self._update_multipart_toggle_button_text() self._update_multipart_toggle_button_text()
if 'proxy_enabled' in settings:
self.settings.setValue(PROXY_ENABLED_KEY, settings['proxy_enabled'])
if 'proxy_host' in settings:
self.settings.setValue(PROXY_HOST_KEY, settings['proxy_host'])
if 'proxy_port' in settings:
self.settings.setValue(PROXY_PORT_KEY, settings['proxy_port'])
if 'proxy_username' in settings:
self.settings.setValue(PROXY_USERNAME_KEY, settings['proxy_username'])
if 'proxy_password' in settings:
self.settings.setValue(PROXY_PASSWORD_KEY, settings['proxy_password'])
self.settings.sync()
def start_multi_threaded_download(self, num_post_workers, **kwargs): def start_multi_threaded_download(self, num_post_workers, **kwargs):
""" """
Initializes and starts the multi-threaded download process. Initializes and starts the multi-threaded download process.
@@ -5205,7 +5460,8 @@ class DownloaderApp (QWidget ):
app_base_dir=worker_args_template.get('app_base_dir'), app_base_dir=worker_args_template.get('app_base_dir'),
manga_filename_style_for_sort_check=worker_args_template.get('manga_filename_style'), manga_filename_style_for_sort_check=worker_args_template.get('manga_filename_style'),
processed_post_ids=worker_args_template.get('processed_post_ids', []), processed_post_ids=worker_args_template.get('processed_post_ids', []),
fetch_all_first=worker_args_template.get('fetch_first', False) fetch_all_first=worker_args_template.get('fetch_first', False),
proxies=worker_args_template.get('proxies')
) )
ppw_expected_keys = list(PostProcessorWorker.__init__.__code__.co_varnames)[1:] ppw_expected_keys = list(PostProcessorWorker.__init__.__code__.co_varnames)[1:]
@@ -5732,9 +5988,7 @@ class DownloaderApp (QWidget ):
if not self.finish_lock.acquire(blocking=False): if not self.finish_lock.acquire(blocking=False):
return return
# --- Flag to track if we still hold the lock ---
lock_held = True lock_held = True
# ----------------------------------------------------
try: try:
if self.is_finishing: if self.is_finishing:
@@ -5743,6 +5997,14 @@ class DownloaderApp (QWidget ):
if cancelled_by_user: if cancelled_by_user:
self.log_signal.emit("✅ Cancellation complete. Resetting UI.") self.log_signal.emit("✅ Cancellation complete. Resetting UI.")
# --- NEW: Reset Queue State on Cancel ---
if getattr(self, 'is_running_job_queue', False):
self.log_signal.emit("🛑 Queue execution stopped by user.")
self.is_running_job_queue = False
self.current_job_file = None
# ----------------------------------------
self._clear_session_file() self._clear_session_file()
self.interrupted_session_data = None self.interrupted_session_data = None
self.is_restore_pending = False self.is_restore_pending = False
@@ -5757,7 +6019,7 @@ class DownloaderApp (QWidget ):
self.log_signal.emit("🏁 Download of current item complete.") self.log_signal.emit("🏁 Download of current item complete.")
# --- QUEUE PROCESSING BLOCK --- # --- EXISTING: FAVORITE QUEUE PROCESSING BLOCK ---
if self.is_processing_favorites_queue and self.favorite_download_queue: if self.is_processing_favorites_queue and self.favorite_download_queue:
self.log_signal.emit("✅ Item finished. Processing next in queue...") self.log_signal.emit("✅ Item finished. Processing next in queue...")
if self.download_thread and isinstance(self.download_thread, QThread): if self.download_thread and isinstance(self.download_thread, QThread):
@@ -5765,13 +6027,46 @@ class DownloaderApp (QWidget ):
self.download_thread = None self.download_thread = None
self.is_finishing = False self.is_finishing = False
# FIX: Manual release + update flag
self.finish_lock.release() self.finish_lock.release()
lock_held = False lock_held = False
self._process_next_favorite_download() self._process_next_favorite_download()
return return
# ---------------------------------------------------------
if getattr(self, 'is_running_job_queue', False) and getattr(self, 'current_job_file', None):
self.log_signal.emit(f"✅ Job finished. Deleting job file: {os.path.basename(self.current_job_file)}")
if self.retryable_failed_files_info:
self.log_signal.emit(f"⚠️ Job had {len(self.retryable_failed_files_info)} incomplete files. Adding to cumulative error report.")
self.permanently_failed_files_for_dialog.extend(self.retryable_failed_files_info)
self._update_error_button_count()
self.retryable_failed_files_info.clear()
self._finalize_download_history()
if self.thread_pool:
self.thread_pool.shutdown(wait=False)
self.thread_pool = None
self._cleanup_temp_files()
self.single_pdf_setting = False # Reset per job
# 2. Delete the finished job file so it isn't run again
try:
if os.path.exists(self.current_job_file):
os.remove(self.current_job_file)
except Exception as e:
self.log_signal.emit(f"⚠️ Failed to delete finished job file: {e}")
# 3. Reset state for next job
self.current_job_file = None
self.is_finishing = False
# 4. Release lock
self.finish_lock.release()
lock_held = False
# 5. Trigger next job in queue (using QTimer to allow stack to unwind)
QTimer.singleShot(100, self._process_next_queued_job)
return
if self.is_processing_favorites_queue: if self.is_processing_favorites_queue:
self.is_processing_favorites_queue = False self.is_processing_favorites_queue = False
@@ -5888,12 +6183,21 @@ class DownloaderApp (QWidget ):
# Reset the finishing lock and exit to let the retry session take over # Reset the finishing lock and exit to let the retry session take over
self.is_finishing = False self.is_finishing = False
# Release lock here as we are returning
self.finish_lock.release()
lock_held = False
return return
self.is_fetcher_thread_running = False self.is_fetcher_thread_running = False
# --- POST DOWNLOAD ACTION (Only if queue is finished or not running queue) ---
if not cancelled_by_user and not self.is_processing_favorites_queue: if not cancelled_by_user and not self.is_processing_favorites_queue:
self._execute_post_download_action() # If we were running a job queue, we only do this when the queue is EMPTY (handled by _process_next_queued_job)
# But since we return early for job queue continuation above, getting here means
# we are either in a standard download OR the job queue has finished/was cancelled.
if not getattr(self, 'is_running_job_queue', False):
self._execute_post_download_action()
self.set_ui_enabled(True) self.set_ui_enabled(True)
self._update_button_states_and_connections() self._update_button_states_and_connections()

View File

@@ -347,7 +347,6 @@ def setup_ui(main_app):
left_layout.addLayout(checkboxes_group_layout) left_layout.addLayout(checkboxes_group_layout)
# --- Action Buttons & Remaining UI --- # --- Action Buttons & Remaining UI ---
# ... (The rest of the setup_ui function remains unchanged)
main_app.standard_action_buttons_widget = QWidget() main_app.standard_action_buttons_widget = QWidget()
btn_layout = QHBoxLayout(main_app.standard_action_buttons_widget) btn_layout = QHBoxLayout(main_app.standard_action_buttons_widget)
btn_layout.setContentsMargins(0, 10, 0, 0) btn_layout.setContentsMargins(0, 10, 0, 0)
@@ -357,6 +356,11 @@ def setup_ui(main_app):
font.setBold(True) font.setBold(True)
main_app.download_btn.setFont(font) main_app.download_btn.setFont(font)
main_app.download_btn.clicked.connect(main_app.start_download) main_app.download_btn.clicked.connect(main_app.start_download)
main_app.add_queue_btn = QPushButton(" Add to Queue")
main_app.add_queue_btn.setToolTip("Save current settings as a job for later execution.")
main_app.add_queue_btn.clicked.connect(main_app.add_current_settings_to_queue)
main_app.pause_btn = QPushButton("⏸️ Pause Download") main_app.pause_btn = QPushButton("⏸️ Pause Download")
main_app.pause_btn.setEnabled(False) main_app.pause_btn.setEnabled(False)
main_app.pause_btn.clicked.connect(main_app._handle_pause_resume_action) main_app.pause_btn.clicked.connect(main_app._handle_pause_resume_action)
@@ -367,6 +371,7 @@ def setup_ui(main_app):
main_app.error_btn.setToolTip("View files skipped due to errors and optionally retry them.") main_app.error_btn.setToolTip("View files skipped due to errors and optionally retry them.")
main_app.error_btn.setEnabled(True) main_app.error_btn.setEnabled(True)
btn_layout.addWidget(main_app.download_btn) btn_layout.addWidget(main_app.download_btn)
btn_layout.addWidget(main_app.add_queue_btn)
btn_layout.addWidget(main_app.pause_btn) btn_layout.addWidget(main_app.pause_btn)
btn_layout.addWidget(main_app.cancel_btn) btn_layout.addWidget(main_app.cancel_btn)
btn_layout.addWidget(main_app.error_btn) btn_layout.addWidget(main_app.error_btn)