mirror of
https://github.com/Yuvi9587/Kemono-Downloader.git
synced 2025-12-29 16:14:44 +00:00
Compare commits
19 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fae9a4bbe2 | ||
|
|
1ad1e53b57 | ||
|
|
77bd428b91 | ||
|
|
4bf57eb752 | ||
|
|
de202961a0 | ||
|
|
e806b6de66 | ||
|
|
cb8dd3b7f3 | ||
|
|
5a8c151c97 | ||
|
|
50ba60a461 | ||
|
|
23521e7060 | ||
|
|
f9c504b936 | ||
|
|
efa0abd0f1 | ||
|
|
7d76d00470 | ||
|
|
1494d3f456 | ||
|
|
675646e763 | ||
|
|
611e892576 | ||
|
|
23fd7f0714 | ||
|
|
cfcd800a49 | ||
|
|
24acec2dc3 |
@@ -68,6 +68,15 @@ DISCORD_TOKEN_KEY = "discord/token"
|
||||
|
||||
POST_DOWNLOAD_ACTION_KEY = "postDownloadAction"
|
||||
|
||||
|
||||
# --- Proxy / Network Keys ---
|
||||
PROXY_ENABLED_KEY = "proxy/enabled"
|
||||
PROXY_HOST_KEY = "proxy/host"
|
||||
PROXY_PORT_KEY = "proxy/port"
|
||||
PROXY_USERNAME_KEY = "proxy/username"
|
||||
PROXY_PASSWORD_KEY = "proxy/password"
|
||||
PROXY_TYPE_KEY = "proxy_type"
|
||||
|
||||
# --- UI Constants and Identifiers ---
|
||||
HTML_PREFIX = "<!HTML!>"
|
||||
LOG_DISPLAY_LINKS = "links"
|
||||
|
||||
@@ -5,7 +5,8 @@ import time
|
||||
import random
|
||||
from urllib.parse import urlparse
|
||||
|
||||
def get_chapter_list(scraper, series_url, logger_func):
|
||||
# 1. Update arguments to accept proxies=None
|
||||
def get_chapter_list(scraper, series_url, logger_func, proxies=None):
|
||||
"""
|
||||
Checks if a URL is a series page and returns a list of all chapter URLs if it is.
|
||||
Relies on a passed-in scraper session for connection.
|
||||
@@ -16,9 +17,13 @@ def get_chapter_list(scraper, series_url, logger_func):
|
||||
response = None
|
||||
max_retries = 8
|
||||
|
||||
# 2. Define smart timeout logic
|
||||
req_timeout = (30, 120) if proxies else 30
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
response = scraper.get(series_url, headers=headers, timeout=30)
|
||||
# 3. Add proxies, verify=False, and the new timeout
|
||||
response = scraper.get(series_url, headers=headers, timeout=req_timeout, proxies=proxies, verify=False)
|
||||
response.raise_for_status()
|
||||
logger_func(f" [AllComic] Successfully connected to series page on attempt {attempt + 1}.")
|
||||
break
|
||||
@@ -53,7 +58,8 @@ def get_chapter_list(scraper, series_url, logger_func):
|
||||
logger_func(f" [AllComic] ❌ Error parsing chapters after successful connection: {e}")
|
||||
return []
|
||||
|
||||
def fetch_chapter_data(scraper, chapter_url, logger_func):
|
||||
# 4. Update arguments here too
|
||||
def fetch_chapter_data(scraper, chapter_url, logger_func, proxies=None):
|
||||
"""
|
||||
Fetches the comic title, chapter title, and image URLs for a single chapter page.
|
||||
Relies on a passed-in scraper session for connection.
|
||||
@@ -64,9 +70,14 @@ def fetch_chapter_data(scraper, chapter_url, logger_func):
|
||||
|
||||
response = None
|
||||
max_retries = 8
|
||||
|
||||
# 5. Define smart timeout logic again
|
||||
req_timeout = (30, 120) if proxies else 30
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
response = scraper.get(chapter_url, headers=headers, timeout=30)
|
||||
# 6. Add proxies, verify=False, and timeout
|
||||
response = scraper.get(chapter_url, headers=headers, timeout=req_timeout, proxies=proxies, verify=False)
|
||||
response.raise_for_status()
|
||||
break
|
||||
except requests.RequestException as e:
|
||||
|
||||
@@ -4,13 +4,37 @@ from urllib.parse import urlparse
|
||||
import json
|
||||
import requests
|
||||
import cloudscraper
|
||||
import ssl
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.poolmanager import PoolManager
|
||||
|
||||
from ..utils.network_utils import extract_post_info, prepare_cookies_for_request
|
||||
from ..config.constants import (
|
||||
STYLE_DATE_POST_TITLE
|
||||
STYLE_DATE_POST_TITLE,
|
||||
STYLE_DATE_BASED,
|
||||
STYLE_POST_TITLE_GLOBAL_NUMBERING
|
||||
)
|
||||
|
||||
# --- NEW: Custom Adapter to fix SSL errors ---
|
||||
class CustomSSLAdapter(HTTPAdapter):
|
||||
"""
|
||||
A custom HTTPAdapter that forces check_hostname=False when using SSL.
|
||||
This prevents the 'Cannot set verify_mode to CERT_NONE' error.
|
||||
"""
|
||||
def init_poolmanager(self, connections, maxsize, block=False):
|
||||
ctx = ssl.create_default_context()
|
||||
# Crucial: Disable hostname checking FIRST, then set verify mode
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
self.poolmanager = PoolManager(
|
||||
num_pools=connections,
|
||||
maxsize=maxsize,
|
||||
block=block,
|
||||
ssl_context=ctx
|
||||
)
|
||||
|
||||
def fetch_posts_paginated(api_url_base, headers, offset, logger, cancellation_event=None, pause_event=None, cookies_dict=None):
|
||||
def fetch_posts_paginated(api_url_base, headers, offset, logger, cancellation_event=None, pause_event=None, cookies_dict=None, proxies=None):
|
||||
"""
|
||||
Fetches a single page of posts from the API with robust retry logic.
|
||||
"""
|
||||
@@ -38,8 +62,11 @@ def fetch_posts_paginated(api_url_base, headers, offset, logger, cancellation_ev
|
||||
log_message += f" (Attempt {attempt + 1}/{max_retries})"
|
||||
logger(log_message)
|
||||
|
||||
request_timeout = (30, 120) if proxies else (15, 60)
|
||||
|
||||
try:
|
||||
with requests.get(paginated_url, headers=headers, timeout=(15, 60), cookies=cookies_dict) as response:
|
||||
with requests.get(paginated_url, headers=headers, timeout=request_timeout, cookies=cookies_dict, proxies=proxies, verify=False) as response:
|
||||
|
||||
response.raise_for_status()
|
||||
response.encoding = 'utf-8'
|
||||
return response.json()
|
||||
@@ -79,39 +106,67 @@ def fetch_posts_paginated(api_url_base, headers, offset, logger, cancellation_ev
|
||||
|
||||
raise RuntimeError(f"Failed to fetch page {paginated_url} after all attempts.")
|
||||
|
||||
def fetch_single_post_data(api_domain, service, user_id, post_id, headers, logger, cookies_dict=None):
|
||||
def fetch_single_post_data(api_domain, service, user_id, post_id, headers, logger, cookies_dict=None, proxies=None):
|
||||
"""
|
||||
--- MODIFIED FUNCTION ---
|
||||
Fetches the full data, including the 'content' field, for a single post using cloudscraper.
|
||||
Includes RETRY logic for 429 Rate Limit errors.
|
||||
"""
|
||||
post_api_url = f"https://{api_domain}/api/v1/{service}/user/{user_id}/post/{post_id}"
|
||||
logger(f" Fetching full content for post ID {post_id}...")
|
||||
|
||||
# FIX: Ensure scraper session is closed after use
|
||||
scraper = None
|
||||
try:
|
||||
scraper = cloudscraper.create_scraper()
|
||||
response = scraper.get(post_api_url, headers=headers, timeout=(15, 300), cookies=cookies_dict)
|
||||
response.raise_for_status()
|
||||
|
||||
full_post_data = response.json()
|
||||
|
||||
if isinstance(full_post_data, list) and full_post_data:
|
||||
return full_post_data[0]
|
||||
if isinstance(full_post_data, dict) and 'post' in full_post_data:
|
||||
return full_post_data['post']
|
||||
return full_post_data
|
||||
|
||||
except Exception as e:
|
||||
logger(f" ❌ Failed to fetch full content for post {post_id}: {e}")
|
||||
return None
|
||||
finally:
|
||||
# CRITICAL FIX: Close the scraper session to free file descriptors and memory
|
||||
if scraper:
|
||||
scraper.close()
|
||||
# Retry settings
|
||||
max_retries = 4
|
||||
|
||||
for attempt in range(max_retries + 1):
|
||||
scraper = None
|
||||
try:
|
||||
scraper = cloudscraper.create_scraper()
|
||||
|
||||
# Mount custom SSL adapter
|
||||
adapter = CustomSSLAdapter()
|
||||
scraper.mount("https://", adapter)
|
||||
|
||||
request_timeout = (30, 300) if proxies else (15, 300)
|
||||
|
||||
response = scraper.get(post_api_url, headers=headers, timeout=request_timeout, cookies=cookies_dict, proxies=proxies, verify=False)
|
||||
|
||||
def fetch_post_comments(api_domain, service, user_id, post_id, headers, logger, cancellation_event=None, pause_event=None, cookies_dict=None):
|
||||
# --- FIX: Handle 429 Rate Limit explicitly ---
|
||||
if response.status_code == 429:
|
||||
wait_time = 20 + (attempt * 10) # 20s, 30s, 40s...
|
||||
logger(f" ⚠️ Rate Limited (429) on post {post_id}. Waiting {wait_time} seconds before retrying...")
|
||||
time.sleep(wait_time)
|
||||
continue # Try loop again
|
||||
# ---------------------------------------------
|
||||
|
||||
response.raise_for_status()
|
||||
|
||||
full_post_data = response.json()
|
||||
|
||||
if isinstance(full_post_data, list) and full_post_data:
|
||||
return full_post_data[0]
|
||||
if isinstance(full_post_data, dict) and 'post' in full_post_data:
|
||||
return full_post_data['post']
|
||||
return full_post_data
|
||||
|
||||
except Exception as e:
|
||||
# Catch "Too Many Requests" if it wasn't caught by status_code check above
|
||||
if "429" in str(e) or "Too Many Requests" in str(e):
|
||||
if attempt < max_retries:
|
||||
wait_time = 20 + (attempt * 10)
|
||||
logger(f" ⚠️ Rate Limit Error caught: {e}. Waiting {wait_time}s...")
|
||||
time.sleep(wait_time)
|
||||
continue
|
||||
|
||||
# Only log error if this was the last attempt
|
||||
if attempt == max_retries:
|
||||
logger(f" ❌ Failed to fetch full content for post {post_id} after {max_retries} retries: {e}")
|
||||
return None
|
||||
finally:
|
||||
if scraper:
|
||||
scraper.close()
|
||||
return None
|
||||
|
||||
def fetch_post_comments(api_domain, service, user_id, post_id, headers, logger, cancellation_event=None, pause_event=None, cookies_dict=None, proxies=None):
|
||||
"""Fetches all comments for a specific post."""
|
||||
if cancellation_event and cancellation_event.is_set():
|
||||
raise RuntimeError("Comment fetch operation cancelled by user.")
|
||||
@@ -120,8 +175,9 @@ def fetch_post_comments(api_domain, service, user_id, post_id, headers, logger,
|
||||
logger(f" Fetching comments: {comments_api_url}")
|
||||
|
||||
try:
|
||||
# FIX: Use context manager
|
||||
with requests.get(comments_api_url, headers=headers, timeout=(10, 30), cookies=cookies_dict) as response:
|
||||
request_timeout = (30, 60) if proxies else (10, 30)
|
||||
|
||||
with requests.get(comments_api_url, headers=headers, timeout=request_timeout, cookies=cookies_dict, proxies=proxies, verify=False) as response:
|
||||
response.raise_for_status()
|
||||
response.encoding = 'utf-8'
|
||||
return response.json()
|
||||
@@ -144,7 +200,8 @@ def download_from_api(
|
||||
app_base_dir=None,
|
||||
manga_filename_style_for_sort_check=None,
|
||||
processed_post_ids=None,
|
||||
fetch_all_first=False
|
||||
fetch_all_first=False,
|
||||
proxies=None
|
||||
):
|
||||
parsed_input_url_for_domain = urlparse(api_url_input)
|
||||
api_domain = parsed_input_url_for_domain.netloc
|
||||
@@ -180,8 +237,9 @@ def download_from_api(
|
||||
direct_post_api_url = f"https://{api_domain}/api/v1/{service}/user/{user_id}/post/{target_post_id}"
|
||||
logger(f" Attempting direct fetch for target post: {direct_post_api_url}")
|
||||
try:
|
||||
# FIX: Use context manager
|
||||
with requests.get(direct_post_api_url, headers=headers, timeout=(10, 30), cookies=cookies_for_api) as direct_response:
|
||||
request_timeout = (30, 60) if proxies else (10, 30)
|
||||
|
||||
with requests.get(direct_post_api_url, headers=headers, timeout=request_timeout, cookies=cookies_for_api, proxies=proxies, verify=False) as direct_response:
|
||||
direct_response.raise_for_status()
|
||||
direct_response.encoding = 'utf-8'
|
||||
direct_post_data = direct_response.json()
|
||||
@@ -208,12 +266,23 @@ def download_from_api(
|
||||
if target_post_id and (start_page or end_page):
|
||||
logger("⚠️ Page range (start/end page) is ignored when a specific post URL is provided (searching all pages for the post).")
|
||||
|
||||
is_manga_mode_fetch_all_and_sort_oldest_first = manga_mode and (manga_filename_style_for_sort_check != STYLE_DATE_POST_TITLE) and not target_post_id
|
||||
# --- FIXED LOGIC HERE ---
|
||||
# Define which styles require fetching ALL posts first (Sequential Mode)
|
||||
styles_requiring_fetch_all = [STYLE_DATE_BASED, STYLE_POST_TITLE_GLOBAL_NUMBERING]
|
||||
|
||||
# Only enable "fetch all and sort" if the current style is explicitly in the list above
|
||||
is_manga_mode_fetch_all_and_sort_oldest_first = (
|
||||
manga_mode and
|
||||
(manga_filename_style_for_sort_check in styles_requiring_fetch_all) and
|
||||
not target_post_id
|
||||
)
|
||||
|
||||
should_fetch_all = fetch_all_first or is_manga_mode_fetch_all_and_sort_oldest_first
|
||||
api_base_url = f"https://{api_domain}/api/v1/{service}/user/{user_id}/posts"
|
||||
page_size = 50
|
||||
|
||||
if is_manga_mode_fetch_all_and_sort_oldest_first:
|
||||
logger(f" Manga Mode (Style: {manga_filename_style_for_sort_check if manga_filename_style_for_sort_check else 'Default'} - Oldest First Sort Active): Fetching all posts to sort by date...")
|
||||
logger(f" Manga Mode (Style: {manga_filename_style_for_sort_check} - Oldest First Sort Active): Fetching all posts to sort by date...")
|
||||
all_posts_for_manga_mode = []
|
||||
current_offset_manga = 0
|
||||
if start_page and start_page > 1:
|
||||
@@ -240,7 +309,7 @@ def download_from_api(
|
||||
logger(f" Manga Mode: Reached specified end page ({end_page}). Stopping post fetch.")
|
||||
break
|
||||
try:
|
||||
posts_batch_manga = fetch_posts_paginated(api_base_url, headers, current_offset_manga, logger, cancellation_event, pause_event, cookies_dict=cookies_for_api)
|
||||
posts_batch_manga = fetch_posts_paginated(api_base_url, headers, current_offset_manga, logger, cancellation_event, pause_event, cookies_dict=cookies_for_api, proxies=proxies)
|
||||
if not isinstance(posts_batch_manga, list):
|
||||
logger(f"❌ API Error (Manga Mode): Expected list of posts, got {type(posts_batch_manga)}.")
|
||||
break
|
||||
@@ -308,8 +377,9 @@ def download_from_api(
|
||||
yield all_posts_for_manga_mode[i:i + page_size]
|
||||
return
|
||||
|
||||
if manga_mode and not target_post_id and (manga_filename_style_for_sort_check == STYLE_DATE_POST_TITLE):
|
||||
logger(f" Manga Mode (Style: {STYLE_DATE_POST_TITLE}): Processing posts in default API order (newest first).")
|
||||
# Log specific message for styles that are in Manga Mode but NOT sorting (Streaming)
|
||||
if manga_mode and not target_post_id and (manga_filename_style_for_sort_check not in styles_requiring_fetch_all):
|
||||
logger(f" Renaming Mode (Style: {manga_filename_style_for_sort_check}): Processing posts in default API order (Streaming).")
|
||||
|
||||
current_page_num = 1
|
||||
current_offset = 0
|
||||
@@ -341,7 +411,7 @@ def download_from_api(
|
||||
break
|
||||
|
||||
try:
|
||||
raw_posts_batch = fetch_posts_paginated(api_base_url, headers, current_offset, logger, cancellation_event, pause_event, cookies_dict=cookies_for_api)
|
||||
raw_posts_batch = fetch_posts_paginated(api_base_url, headers, current_offset, logger, cancellation_event, pause_event, cookies_dict=cookies_for_api, proxies=proxies)
|
||||
if not isinstance(raw_posts_batch, list):
|
||||
logger(f"❌ API Error: Expected list of posts, got {type(raw_posts_batch)} at page {current_page_num} (offset {current_offset}).")
|
||||
break
|
||||
|
||||
@@ -11,11 +11,28 @@ class DeviantArtClient:
|
||||
CLIENT_SECRET = "76b08c69cfb27f26d6161f9ab6d061a1"
|
||||
BASE_API = "https://www.deviantart.com/api/v1/oauth2"
|
||||
|
||||
def __init__(self, logger_func=print):
|
||||
# 1. Accept proxies in init
|
||||
def __init__(self, logger_func=print, proxies=None):
|
||||
self.session = requests.Session()
|
||||
|
||||
# 2. Configure Session with Proxy & SSL settings immediately
|
||||
if proxies:
|
||||
self.session.proxies.update(proxies)
|
||||
self.session.verify = False # Ignore SSL for proxies
|
||||
self.proxies_enabled = True
|
||||
else:
|
||||
self.proxies_enabled = False
|
||||
|
||||
self.session.headers.update({
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
||||
'Accept': '*/*',
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
"Connection": "keep-alive",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
})
|
||||
self.access_token = None
|
||||
self.logger = logger_func
|
||||
@@ -33,7 +50,10 @@ class DeviantArtClient:
|
||||
"client_id": self.CLIENT_ID,
|
||||
"client_secret": self.CLIENT_SECRET
|
||||
}
|
||||
resp = self.session.post(url, data=data, timeout=10)
|
||||
# 3. Smart timeout (longer if proxy)
|
||||
req_timeout = 30 if self.proxies_enabled else 10
|
||||
|
||||
resp = self.session.post(url, data=data, timeout=req_timeout)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
self.access_token = data.get("access_token")
|
||||
@@ -55,12 +75,28 @@ class DeviantArtClient:
|
||||
retries = 0
|
||||
max_retries = 4
|
||||
backoff_delay = 2
|
||||
|
||||
# 4. Smart timeout
|
||||
req_timeout = 30 if self.proxies_enabled else 20
|
||||
|
||||
while True:
|
||||
try:
|
||||
resp = self.session.get(url, params=params, timeout=20)
|
||||
|
||||
# Handle Token Expiration (401)
|
||||
resp = self.session.get(url, params=params, timeout=req_timeout)
|
||||
|
||||
# 429: Rate Limit
|
||||
if resp.status_code == 429:
|
||||
retry_after = resp.headers.get('Retry-After')
|
||||
if retry_after:
|
||||
sleep_time = int(retry_after) + 2 # Add buffer
|
||||
else:
|
||||
# 5. Increase default wait time for 429s
|
||||
sleep_time = 15
|
||||
|
||||
self._log_once(sleep_time, f" [DeviantArt] ⚠️ Rate limit (429). Sleeping {sleep_time}s...")
|
||||
time.sleep(sleep_time)
|
||||
continue
|
||||
|
||||
# 401: Token Expired (Refresh and Retry)
|
||||
if resp.status_code == 401:
|
||||
self.logger(" [DeviantArt] Token expired. Refreshing...")
|
||||
if self.authenticate():
|
||||
@@ -69,64 +105,47 @@ class DeviantArtClient:
|
||||
else:
|
||||
raise Exception("Failed to refresh token")
|
||||
|
||||
# Handle Rate Limiting (429)
|
||||
if resp.status_code == 429:
|
||||
if retries < max_retries:
|
||||
retry_after = resp.headers.get('Retry-After')
|
||||
|
||||
if retry_after:
|
||||
sleep_time = int(retry_after) + 1
|
||||
msg = f" [DeviantArt] ⚠️ Rate limit (Server says wait {sleep_time}s)."
|
||||
else:
|
||||
sleep_time = backoff_delay * (2 ** retries)
|
||||
msg = f" [DeviantArt] ⚠️ Rate limit reached. Retrying in {sleep_time}s..."
|
||||
|
||||
# --- THREAD-SAFE LOGGING CHECK ---
|
||||
should_log = False
|
||||
with self.log_lock:
|
||||
if sleep_time not in self.logged_waits:
|
||||
self.logged_waits.add(sleep_time)
|
||||
should_log = True
|
||||
|
||||
if should_log:
|
||||
self.logger(msg)
|
||||
|
||||
time.sleep(sleep_time)
|
||||
retries += 1
|
||||
continue
|
||||
else:
|
||||
resp.raise_for_status()
|
||||
if 400 <= resp.status_code < 500:
|
||||
resp.raise_for_status()
|
||||
|
||||
if 500 <= resp.status_code < 600:
|
||||
resp.raise_for_status()
|
||||
|
||||
resp.raise_for_status()
|
||||
|
||||
# Clear log history on success so we get warned again if limits return later
|
||||
with self.log_lock:
|
||||
if self.logged_waits:
|
||||
self.logged_waits.clear()
|
||||
self.logged_waits.clear()
|
||||
|
||||
return resp.json()
|
||||
|
||||
except requests.exceptions.HTTPError as e:
|
||||
if e.response is not None and 400 <= e.response.status_code < 500:
|
||||
raise e
|
||||
pass
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
if retries < max_retries:
|
||||
# Using the lock here too to prevent connection error spam
|
||||
should_log = False
|
||||
with self.log_lock:
|
||||
if "conn_error" not in self.logged_waits:
|
||||
self.logged_waits.add("conn_error")
|
||||
should_log = True
|
||||
|
||||
if should_log:
|
||||
self.logger(f" [DeviantArt] Connection error: {e}. Retrying...")
|
||||
|
||||
time.sleep(2)
|
||||
self._log_once("conn_error", f" [DeviantArt] Connection error: {e}. Retrying...")
|
||||
time.sleep(backoff_delay)
|
||||
retries += 1
|
||||
continue
|
||||
raise e
|
||||
|
||||
def _log_once(self, key, message):
|
||||
"""Helper to avoid spamming the same log message during loops."""
|
||||
should_log = False
|
||||
with self.log_lock:
|
||||
if key not in self.logged_waits:
|
||||
self.logged_waits.add(key)
|
||||
should_log = True
|
||||
if should_log:
|
||||
self.logger(message)
|
||||
|
||||
def get_deviation_uuid(self, url):
|
||||
"""Scrapes the deviation page to find the UUID."""
|
||||
try:
|
||||
resp = self.session.get(url, timeout=15)
|
||||
req_timeout = 30 if self.proxies_enabled else 15
|
||||
resp = self.session.get(url, timeout=req_timeout)
|
||||
match = re.search(r'"deviationUuid":"([^"]+)"', resp.text)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
0
src/core/hentaifox.txt
Normal file
0
src/core/hentaifox.txt
Normal file
60
src/core/hentaifox_client.py
Normal file
60
src/core/hentaifox_client.py
Normal file
@@ -0,0 +1,60 @@
|
||||
import requests
|
||||
import re
|
||||
from bs4 import BeautifulSoup # Optional, but regex is faster for this specific site
|
||||
|
||||
# Logic derived from NHdownloader.sh 'hentaifox' function
|
||||
BASE_URL = "https://hentaifox.com"
|
||||
HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
|
||||
"Referer": "https://hentaifox.com/",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"
|
||||
}
|
||||
|
||||
def get_gallery_id(url_or_id):
|
||||
"""Extracts numbers from URL or returns the ID string."""
|
||||
match = re.search(r"(\d+)", str(url_or_id))
|
||||
return match.group(1) if match else None
|
||||
|
||||
def get_gallery_metadata(gallery_id):
|
||||
"""
|
||||
Fetches the main gallery page to get the Title and Total Pages.
|
||||
Equivalent to the first part of the 'hentaifox' function in .sh file.
|
||||
"""
|
||||
url = f"{BASE_URL}/gallery/{gallery_id}/"
|
||||
response = requests.get(url, headers=HEADERS)
|
||||
response.raise_for_status()
|
||||
html = response.text
|
||||
|
||||
# Extract Title (Bash: grep -o '<title>.*</title>')
|
||||
title_match = re.search(r'<title>(.*?)</title>', html)
|
||||
title = title_match.group(1).replace(" - HentaiFox", "").strip() if title_match else f"Gallery {gallery_id}"
|
||||
|
||||
# Extract Total Pages (Bash: grep -Eo 'Pages: [0-9]*')
|
||||
pages_match = re.search(r'Pages: (\d+)', html)
|
||||
if not pages_match:
|
||||
raise ValueError("Could not find total pages count.")
|
||||
|
||||
total_pages = int(pages_match.group(1))
|
||||
|
||||
return {
|
||||
"id": gallery_id,
|
||||
"title": title,
|
||||
"total_pages": total_pages
|
||||
}
|
||||
|
||||
def get_image_link_for_page(gallery_id, page_num):
|
||||
"""
|
||||
Fetches the specific reader page to find the actual image URL.
|
||||
Equivalent to the loop in the 'hentaifox' function:
|
||||
url="https://hentaifox.com/g/${id}/${i}/"
|
||||
"""
|
||||
url = f"{BASE_URL}/g/{gallery_id}/{page_num}/"
|
||||
response = requests.get(url, headers=HEADERS)
|
||||
|
||||
# Extract image source (Bash: grep -Eo 'data-src="..."')
|
||||
# Regex looks for: data-src="https://..."
|
||||
match = re.search(r'data-src="(https://[^"]+)"', response.text)
|
||||
|
||||
if match:
|
||||
return match.group(1)
|
||||
return None
|
||||
@@ -3,7 +3,7 @@ import time
|
||||
import os
|
||||
import json
|
||||
import traceback
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed, Future
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed, Future, CancelledError
|
||||
from .api_client import download_from_api
|
||||
from .workers import PostProcessorWorker
|
||||
from ..config.constants import (
|
||||
@@ -84,8 +84,18 @@ class DownloadManager:
|
||||
|
||||
is_single_post = bool(config.get('target_post_id_from_initial_url'))
|
||||
use_multithreading = config.get('use_multithreading', True)
|
||||
is_manga_sequential = config.get('manga_mode_active') and config.get('manga_filename_style') in [STYLE_DATE_BASED, STYLE_POST_TITLE_GLOBAL_NUMBERING]
|
||||
|
||||
# --- FIXED LOGIC: Strict check for sequential fetch modes ---
|
||||
# Only "Date Based" and "Title + Global Numbering" require fetching the full list first.
|
||||
# "Custom", "Date + Title", "Original Name", and "Post ID" will now use the pool (streaming).
|
||||
sequential_styles = [STYLE_DATE_BASED, STYLE_POST_TITLE_GLOBAL_NUMBERING]
|
||||
|
||||
is_manga_sequential = (
|
||||
config.get('manga_mode_active') and
|
||||
config.get('manga_filename_style') in sequential_styles
|
||||
)
|
||||
|
||||
# If it is NOT a strictly sequential manga mode, we use the pool (fetch-as-we-go)
|
||||
should_use_multithreading_for_posts = use_multithreading and not is_single_post and not is_manga_sequential
|
||||
|
||||
if should_use_multithreading_for_posts:
|
||||
@@ -97,12 +107,34 @@ class DownloadManager:
|
||||
fetcher_thread.start()
|
||||
else:
|
||||
# Single-threaded mode does not use the manager's complex logic
|
||||
self._log("ℹ️ Manager is handing off to a single-threaded worker...")
|
||||
self._log("ℹ️ Manager is handing off to a single-threaded worker (Sequential Mode)...")
|
||||
# The single-threaded worker will manage its own lifecycle and signals.
|
||||
# The manager's role for this session is effectively over.
|
||||
self.is_running = False # Allow another session to start if needed
|
||||
self.progress_queue.put({'type': 'handoff_to_single_thread', 'payload': (config,)})
|
||||
|
||||
def _get_proxies_from_config(self, config):
|
||||
"""Constructs the proxy dictionary from the config."""
|
||||
if not config.get('proxy_enabled'):
|
||||
return None
|
||||
|
||||
host = config.get('proxy_host')
|
||||
port = config.get('proxy_port')
|
||||
if not host or not port:
|
||||
return None
|
||||
|
||||
proxy_str = f"http://{host}:{port}"
|
||||
|
||||
# Add auth if provided
|
||||
user = config.get('proxy_username')
|
||||
password = config.get('proxy_password')
|
||||
if user and password:
|
||||
proxy_str = f"http://{user}:{password}@{host}:{port}"
|
||||
|
||||
return {
|
||||
"http": proxy_str,
|
||||
"https": proxy_str
|
||||
}
|
||||
|
||||
def _fetch_and_queue_posts_for_pool(self, config, restore_data, creator_profile_data):
|
||||
"""
|
||||
@@ -117,6 +149,9 @@ class DownloadManager:
|
||||
session_processed_ids = set(restore_data.get('processed_post_ids', [])) if restore_data else set()
|
||||
profile_processed_ids = set(creator_profile_data.get('processed_post_ids', []))
|
||||
processed_ids = session_processed_ids.union(profile_processed_ids)
|
||||
|
||||
# Helper to get proxies
|
||||
proxies = self._get_proxies_from_config(config)
|
||||
|
||||
if restore_data and 'all_posts_data' in restore_data:
|
||||
# This logic for session restore remains as it relies on a pre-fetched list
|
||||
@@ -132,127 +167,113 @@ class DownloadManager:
|
||||
return
|
||||
|
||||
for post_data in posts_to_process:
|
||||
if self.cancellation_event.is_set(): break
|
||||
worker = PostProcessorWorker(post_data, config, self.progress_queue)
|
||||
if self.cancellation_event.is_set():
|
||||
break
|
||||
|
||||
worker_args = self._map_config_to_worker_args(post_data, config)
|
||||
# Manually inject proxies here if _map_config_to_worker_args didn't catch it (though it should)
|
||||
worker_args['proxies'] = proxies
|
||||
|
||||
worker = PostProcessorWorker(**worker_args)
|
||||
future = self.thread_pool.submit(worker.process)
|
||||
future.add_done_callback(self._handle_future_result)
|
||||
self.active_futures.append(future)
|
||||
else:
|
||||
# --- START: REFACTORED STREAMING LOGIC ---
|
||||
# --- Streaming Logic ---
|
||||
if proxies:
|
||||
self._log(f" 🌐 Using Proxy: {config.get('proxy_host')}:{config.get('proxy_port')}")
|
||||
|
||||
post_generator = download_from_api(
|
||||
api_url_input=config['api_url'],
|
||||
logger=self._log,
|
||||
start_page=config.get('start_page'),
|
||||
end_page=config.get('end_page'),
|
||||
manga_mode=config.get('manga_mode_active', False),
|
||||
cancellation_event=self.cancellation_event,
|
||||
pause_event=self.pause_event,
|
||||
use_cookie=config.get('use_cookie', False),
|
||||
cookie_text=config.get('cookie_text', ''),
|
||||
selected_cookie_file=config.get('selected_cookie_file'),
|
||||
app_base_dir=config.get('app_base_dir'),
|
||||
manga_filename_style_for_sort_check=config.get('manga_filename_style'),
|
||||
processed_post_ids=list(processed_ids)
|
||||
cookies_dict=None, # Cookie handling handled inside client if needed
|
||||
proxies=proxies # <--- NEW: Pass proxies to API client
|
||||
)
|
||||
|
||||
self.total_posts = 0
|
||||
self.processed_posts = 0
|
||||
|
||||
# Process posts in batches as they are yielded by the API client
|
||||
for batch in post_generator:
|
||||
for post_batch in post_generator:
|
||||
if self.cancellation_event.is_set():
|
||||
self._log(" Post fetching cancelled.")
|
||||
break
|
||||
|
||||
# Filter out any posts that might have been processed since the start
|
||||
posts_in_batch_to_process = [p for p in batch if p.get('id') not in processed_ids]
|
||||
|
||||
if not posts_in_batch_to_process:
|
||||
if not post_batch:
|
||||
continue
|
||||
|
||||
# Update total count and immediately inform the UI
|
||||
self.total_posts += len(posts_in_batch_to_process)
|
||||
self.progress_queue.put({'type': 'overall_progress', 'payload': (self.total_posts, self.processed_posts)})
|
||||
new_posts_batch = [p for p in post_batch if p.get('id') not in processed_ids]
|
||||
|
||||
if not new_posts_batch:
|
||||
continue
|
||||
|
||||
for post_data in posts_in_batch_to_process:
|
||||
if self.cancellation_event.is_set(): break
|
||||
worker = PostProcessorWorker(post_data, config, self.progress_queue)
|
||||
# Update total posts dynamically as we find them
|
||||
self.total_posts += len(new_posts_batch)
|
||||
|
||||
for post_data in new_posts_batch:
|
||||
if self.cancellation_event.is_set():
|
||||
break
|
||||
|
||||
# MAPPING CONFIG TO WORKER ARGS
|
||||
worker_args = self._map_config_to_worker_args(post_data, config)
|
||||
worker = PostProcessorWorker(**worker_args)
|
||||
|
||||
future = self.thread_pool.submit(worker.process)
|
||||
future.add_done_callback(self._handle_future_result)
|
||||
self.active_futures.append(future)
|
||||
|
||||
if self.total_posts == 0 and not self.cancellation_event.is_set():
|
||||
self._log("✅ No new posts found to process.")
|
||||
|
||||
# Small sleep to prevent UI freeze
|
||||
time.sleep(0.01)
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"❌ CRITICAL ERROR in post fetcher thread: {e}")
|
||||
self._log(traceback.format_exc())
|
||||
self._log(f"❌ Critical Error in Fetcher Thread: {e}")
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
if self.thread_pool:
|
||||
self.thread_pool.shutdown(wait=True)
|
||||
self.is_running = False
|
||||
self._log("🏁 All processing tasks have completed or been cancelled.")
|
||||
self.progress_queue.put({
|
||||
'type': 'finished',
|
||||
'payload': (self.total_downloads, self.total_skips, self.cancellation_event.is_set(), self.all_kept_original_filenames)
|
||||
})
|
||||
self.is_running = False # Mark as not running so we can finish
|
||||
# The main window checks active futures, so we just exit this thread.
|
||||
|
||||
def _handle_future_result(self, future: Future):
|
||||
"""Callback executed when a worker task completes."""
|
||||
if self.cancellation_event.is_set():
|
||||
return
|
||||
|
||||
with threading.Lock(): # Protect shared counters
|
||||
self.processed_posts += 1
|
||||
try:
|
||||
if future.cancelled():
|
||||
self._log("⚠️ A post processing task was cancelled.")
|
||||
self.total_skips += 1
|
||||
else:
|
||||
result = future.result()
|
||||
(dl_count, skip_count, kept_originals,
|
||||
retryable, permanent, history) = result
|
||||
self.total_downloads += dl_count
|
||||
self.total_skips += skip_count
|
||||
self.all_kept_original_filenames.extend(kept_originals)
|
||||
if retryable:
|
||||
self.progress_queue.put({'type': 'retryable_failure', 'payload': (retryable,)})
|
||||
if permanent:
|
||||
self.progress_queue.put({'type': 'permanent_failure', 'payload': (permanent,)})
|
||||
if history:
|
||||
self.progress_queue.put({'type': 'post_processed_history', 'payload': (history,)})
|
||||
post_id = history.get('post_id')
|
||||
if post_id and self.current_creator_profile_path:
|
||||
profile_data = self._setup_creator_profile({'creator_name_for_profile': self.current_creator_name_for_profile, 'session_file_path': self.session_file_path})
|
||||
if post_id not in profile_data.get('processed_post_ids', []):
|
||||
profile_data.setdefault('processed_post_ids', []).append(post_id)
|
||||
self._save_creator_profile(profile_data)
|
||||
|
||||
except Exception as e:
|
||||
self._log(f"❌ Worker task resulted in an exception: {e}")
|
||||
self.total_skips += 1 # Count errored posts as skipped
|
||||
self.progress_queue.put({'type': 'overall_progress', 'payload': (self.total_posts, self.processed_posts)})
|
||||
def _map_config_to_worker_args(self, post_data, config):
|
||||
"""Helper to map the flat config dict to PostProcessorWorker arguments."""
|
||||
# Get proxy dict
|
||||
proxies = self._get_proxies_from_config(config)
|
||||
|
||||
# This mirrors the arguments in workers.py PostProcessorWorker.__init__
|
||||
return {
|
||||
'post_data': post_data,
|
||||
'download_root': config.get('output_dir'),
|
||||
'known_names': [], # If needed, pass KNOWN_NAMES or load them
|
||||
'filter_character_list': [], # Parsed filters if available in config
|
||||
'emitter': self.progress_queue,
|
||||
'unwanted_keywords': set(), # Parse if needed
|
||||
'filter_mode': config.get('filter_mode'),
|
||||
'skip_zip': config.get('skip_zip'),
|
||||
'use_subfolders': config.get('use_subfolders'),
|
||||
'use_post_subfolders': config.get('use_post_subfolders'),
|
||||
'target_post_id_from_initial_url': config.get('target_post_id_from_initial_url'),
|
||||
'custom_folder_name': config.get('custom_folder_name'),
|
||||
'compress_images': config.get('compress_images'),
|
||||
'download_thumbnails': config.get('download_thumbnails'),
|
||||
'service': config.get('service') or 'unknown',
|
||||
'user_id': config.get('user_id') or 'unknown',
|
||||
'pause_event': self.pause_event,
|
||||
'api_url_input': config.get('api_url'),
|
||||
'cancellation_event': self.cancellation_event,
|
||||
'downloaded_files': None,
|
||||
'downloaded_file_hashes': None,
|
||||
'downloaded_files_lock': None,
|
||||
'downloaded_file_hashes_lock': None,
|
||||
'manga_mode_active': config.get('manga_mode_active'),
|
||||
'manga_filename_style': config.get('manga_filename_style'),
|
||||
'manga_custom_filename_format': config.get('custom_manga_filename_format', "{published} {title}"),
|
||||
'manga_custom_date_format': config.get('manga_custom_date_format', "YYYY-MM-DD"),
|
||||
'use_multithreading': config.get('use_multithreading', True),
|
||||
'proxies': proxies, # <--- NEW: Pass proxies to worker
|
||||
}
|
||||
|
||||
def _setup_creator_profile(self, config):
|
||||
"""Prepares the path and loads data for the current creator's profile."""
|
||||
self.current_creator_name_for_profile = config.get('creator_name_for_profile')
|
||||
if not self.current_creator_name_for_profile:
|
||||
self._log("⚠️ Cannot create creator profile: Name not provided in config.")
|
||||
return {}
|
||||
|
||||
appdata_dir = os.path.dirname(config.get('session_file_path', '.'))
|
||||
self.creator_profiles_dir = os.path.join(appdata_dir, "creator_profiles")
|
||||
os.makedirs(self.creator_profiles_dir, exist_ok=True)
|
||||
|
||||
safe_filename = clean_folder_name(self.current_creator_name_for_profile) + ".json"
|
||||
self.current_creator_profile_path = os.path.join(self.creator_profiles_dir, safe_filename)
|
||||
|
||||
if os.path.exists(self.current_creator_profile_path):
|
||||
try:
|
||||
with open(self.current_creator_profile_path, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
self._log(f"❌ Error loading creator profile '{safe_filename}': {e}. Starting fresh.")
|
||||
# Extract name logic here or assume config has it
|
||||
self.current_creator_name_for_profile = "Unknown"
|
||||
# You should ideally extract name from URL or config here if available
|
||||
return {}
|
||||
|
||||
def _save_creator_profile(self, data):
|
||||
@@ -280,6 +301,33 @@ class DownloadManager:
|
||||
self.cancellation_event.set()
|
||||
|
||||
if self.thread_pool:
|
||||
self._log(" Signaling all worker threads to stop and shutting down pool...")
|
||||
self.thread_pool.shutdown(wait=False)
|
||||
self.thread_pool.shutdown(wait=False, cancel_futures=True)
|
||||
|
||||
def _handle_future_result(self, future):
|
||||
"""Callback for when a worker task finishes."""
|
||||
if self.active_futures:
|
||||
try:
|
||||
self.active_futures.remove(future)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
try:
|
||||
result = future.result()
|
||||
# result tuple: (download_count, skip_count, kept_original_filenames, ...)
|
||||
if result:
|
||||
self.total_downloads += result[0]
|
||||
self.total_skips += result[1]
|
||||
if len(result) > 3 and result[3]:
|
||||
# filename was kept original
|
||||
pass
|
||||
except CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
self._log(f"❌ Worker Error: {e}")
|
||||
|
||||
self.processed_posts += 1
|
||||
self.progress_queue.put({'type': 'overall_progress', 'payload': (self.total_posts, self.processed_posts)})
|
||||
|
||||
if not self.active_futures and not self.is_running:
|
||||
self._log("✅ All tasks completed.")
|
||||
self.progress_queue.put({'type': 'worker_finished', 'payload': (self.total_downloads, self.total_skips, [], [])})
|
||||
@@ -1,31 +1,35 @@
|
||||
import requests
|
||||
import cloudscraper
|
||||
import json
|
||||
|
||||
def fetch_nhentai_gallery(gallery_id, logger=print):
|
||||
# 1. Update arguments to accept proxies=None
|
||||
def fetch_nhentai_gallery(gallery_id, logger=print, proxies=None):
|
||||
"""
|
||||
Fetches the metadata for a single nhentai gallery using cloudscraper to bypass Cloudflare.
|
||||
|
||||
Args:
|
||||
gallery_id (str or int): The ID of the nhentai gallery.
|
||||
logger (function): A function to log progress and error messages.
|
||||
|
||||
Returns:
|
||||
dict: A dictionary containing the gallery's metadata if successful, otherwise None.
|
||||
Fetches the metadata for a single nhentai gallery.
|
||||
Switched to standard requests to support proxies with self-signed certs.
|
||||
"""
|
||||
api_url = f"https://nhentai.net/api/gallery/{gallery_id}"
|
||||
|
||||
scraper = cloudscraper.create_scraper()
|
||||
# 2. Use a real User-Agent to avoid immediate blocking
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
|
||||
}
|
||||
|
||||
logger(f" Fetching nhentai gallery metadata from: {api_url}")
|
||||
|
||||
# 3. Smart timeout logic
|
||||
req_timeout = (30, 120) if proxies else 20
|
||||
|
||||
try:
|
||||
# Use the scraper to make the GET request
|
||||
response = scraper.get(api_url, timeout=20)
|
||||
# 4. Use requests.get with proxies, verify=False, and timeout
|
||||
response = requests.get(api_url, headers=headers, timeout=req_timeout, proxies=proxies, verify=False)
|
||||
|
||||
if response.status_code == 404:
|
||||
logger(f" ❌ Gallery not found (404): ID {gallery_id}")
|
||||
return None
|
||||
elif response.status_code == 403:
|
||||
logger(f" ❌ Access Denied (403): Cloudflare blocked the request. Try a different proxy or User-Agent.")
|
||||
return None
|
||||
|
||||
response.raise_for_status()
|
||||
|
||||
@@ -36,9 +40,9 @@ def fetch_nhentai_gallery(gallery_id, logger=print):
|
||||
gallery_data['pages'] = gallery_data.pop('images')['pages']
|
||||
return gallery_data
|
||||
else:
|
||||
logger(" ❌ API response is missing essential keys (id, media_id, or images).")
|
||||
logger(" ❌ API response is missing essential keys (id, media_id, images).")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger(f" ❌ An error occurred while fetching gallery {gallery_id}: {e}")
|
||||
logger(f" ❌ Error fetching nhentai metadata: {e}")
|
||||
return None
|
||||
@@ -62,7 +62,8 @@ def robust_clean_name(name):
|
||||
"""A more robust function to remove illegal characters for filenames and folders."""
|
||||
if not name:
|
||||
return ""
|
||||
illegal_chars_pattern = r'[\x00-\x1f<>:"/\\|?*\']'
|
||||
# FIX: Removed \' from the list so apostrophes are kept
|
||||
illegal_chars_pattern = r'[\x00-\x1f<>:"/\\|?*]'
|
||||
cleaned_name = re.sub(illegal_chars_pattern, '', name)
|
||||
|
||||
cleaned_name = cleaned_name.strip(' .')
|
||||
@@ -133,7 +134,8 @@ class PostProcessorWorker:
|
||||
sfp_threshold=None,
|
||||
handle_unknown_mode=False,
|
||||
creator_name_cache=None,
|
||||
add_info_in_pdf=False
|
||||
add_info_in_pdf=False,
|
||||
proxies=None
|
||||
|
||||
):
|
||||
self.post = post_data
|
||||
@@ -208,9 +210,8 @@ class PostProcessorWorker:
|
||||
self.sfp_threshold = sfp_threshold
|
||||
self.handle_unknown_mode = handle_unknown_mode
|
||||
self.creator_name_cache = creator_name_cache
|
||||
#-- New assign --
|
||||
self.add_info_in_pdf = add_info_in_pdf
|
||||
#-- New assign --
|
||||
self.proxies = proxies
|
||||
|
||||
|
||||
if self.compress_images and Image is None:
|
||||
@@ -263,7 +264,7 @@ class PostProcessorWorker:
|
||||
new_url = parsed_url._replace(netloc=new_domain).geturl()
|
||||
|
||||
try:
|
||||
with requests.head(new_url, headers={'User-Agent': 'Mozilla/5.0'}, timeout=5, allow_redirects=True) as resp:
|
||||
with requests.head(new_url, headers={'User-Agent': 'Mozilla/5.0'}, timeout=5, allow_redirects=True, proxies=self.proxies, verify=False) as resp:
|
||||
if resp.status_code == 200:
|
||||
return new_url
|
||||
except requests.RequestException:
|
||||
@@ -338,7 +339,8 @@ class PostProcessorWorker:
|
||||
api_original_filename_for_size_check = file_info.get('_original_name_for_log', file_info.get('name'))
|
||||
try:
|
||||
# Use a stream=True HEAD request to get headers without downloading the body
|
||||
with requests.head(file_url, headers=file_download_headers, timeout=15, cookies=cookies_to_use_for_file, allow_redirects=True) as head_response:
|
||||
with requests.head(file_url, headers=file_download_headers, timeout=15, cookies=cookies_to_use_for_file, allow_redirects=True, proxies=self.proxies, verify=False) as head_response:
|
||||
|
||||
head_response.raise_for_status()
|
||||
content_length = head_response.headers.get('Content-Length')
|
||||
if content_length:
|
||||
@@ -672,7 +674,7 @@ class PostProcessorWorker:
|
||||
|
||||
current_url_to_try = file_url
|
||||
|
||||
response = requests.get(current_url_to_try, headers=file_download_headers, timeout=(30, 300), stream=True, cookies=cookies_to_use_for_file)
|
||||
response = requests.get(current_url_to_try, headers=file_download_headers, timeout=(30, 300), stream=True, cookies=cookies_to_use_for_file, proxies=self.proxies, verify=False)
|
||||
|
||||
if response.status_code == 403 and ('kemono.' in current_url_to_try or 'coomer.' in current_url_to_try):
|
||||
self.logger(f" ⚠️ Got 403 Forbidden for '{api_original_filename}'. Attempting subdomain rotation...")
|
||||
@@ -681,8 +683,7 @@ class PostProcessorWorker:
|
||||
self.logger(f" Retrying with new URL: {new_url}")
|
||||
file_url = new_url
|
||||
response.close() # Close the old response
|
||||
response = requests.get(new_url, headers=file_download_headers, timeout=(30, 300), stream=True, cookies=cookies_to_use_for_file)
|
||||
|
||||
response = requests.get(new_url, headers=file_download_headers, timeout=(30, 300), stream=True, cookies=cookies_to_use_for_file, proxies=self.proxies, verify=False)
|
||||
response.raise_for_status()
|
||||
|
||||
# --- REVISED AND MOVED SIZE CHECK LOGIC ---
|
||||
@@ -1104,8 +1105,8 @@ class PostProcessorWorker:
|
||||
'Referer': creator_page_url,
|
||||
'Accept': 'text/css'
|
||||
}
|
||||
cookies = prepare_cookies_for_request(self.use_cookie, self.cookie_text, self.selected_cookie_file, self.app_base_dir, self.logger, target_domain=api_domain)
|
||||
full_post_data = fetch_single_post_data(api_domain, self.service, self.user_id, post_id, headers, self.logger, cookies_dict=cookies)
|
||||
cookies = prepare_cookies_for_request(self.use_cookie, self.cookie_text, self.selected_cookie_file, self.app_base_dir, self.logger, target_domain=api_domain)
|
||||
full_post_data = fetch_single_post_data(api_domain, self.service, self.user_id, post_id, headers, self.logger, cookies_dict=cookies, proxies=self.proxies)
|
||||
if full_post_data:
|
||||
self.logger(" ✅ Full post data fetched successfully.")
|
||||
self.post = full_post_data
|
||||
@@ -1306,13 +1307,17 @@ class PostProcessorWorker:
|
||||
if not any(d in api_domain_for_comments.lower() for d in ['kemono.su', 'kemono.party', 'kemono.cr', 'coomer.su', 'coomer.party', 'coomer.st']):
|
||||
self.logger(f"⚠️ Unrecognized domain '{api_domain_for_comments}' for comment API. Defaulting based on service.")
|
||||
api_domain_for_comments = "kemono.cr" if "kemono" in self.service.lower() else "coomer.st"
|
||||
|
||||
# Fetch comments (Indented correctly now)
|
||||
comments_data = fetch_post_comments(
|
||||
api_domain_for_comments, self.service, self.user_id, post_id,
|
||||
headers, self.logger, self.cancellation_event, self.pause_event,
|
||||
cookies_dict=prepare_cookies_for_request(
|
||||
self.use_cookie, self.cookie_text, self.selected_cookie_file, self.app_base_dir, self.logger
|
||||
)
|
||||
),
|
||||
proxies=self.proxies
|
||||
)
|
||||
|
||||
if comments_data:
|
||||
self.logger(f" Fetched {len(comments_data)} comments for post {post_id}.")
|
||||
for comment_item_idx, comment_item in enumerate(comments_data):
|
||||
@@ -1339,8 +1344,8 @@ class PostProcessorWorker:
|
||||
except RuntimeError as e_fetch_comment:
|
||||
self.logger(f" ⚠️ Error fetching or processing comments for post {post_id}: {e_fetch_comment}")
|
||||
except Exception as e_generic_comment:
|
||||
self.logger(f" ❌ Unexpected error during comment processing for post {post_id}: {e_generic_comment}\n{traceback.format_exc(limit=2)}")
|
||||
self.logger(f" [Char Scope: Comments] Phase 2 Result: post_is_candidate_by_comment_char_match = {post_is_candidate_by_comment_char_match}")
|
||||
self.logger(f" ❌ Unexpected error during comment processing for post {post_id}: {e_generic_comment}\n{traceback.format_exc(limit=2)}")
|
||||
|
||||
else:
|
||||
self.logger(f" [Char Scope: Comments] Phase 2: Skipped comment check for post ID '{post_id}' because a file match already made it a candidate.")
|
||||
|
||||
@@ -1595,12 +1600,11 @@ class PostProcessorWorker:
|
||||
|
||||
should_create_post_subfolder = self.use_post_subfolders
|
||||
|
||||
if (not self.use_post_subfolders and self.use_subfolders and
|
||||
if (not self.use_post_subfolders and
|
||||
self.sfp_threshold is not None and num_potential_files_in_post >= self.sfp_threshold):
|
||||
|
||||
self.logger(f" ℹ️ Post has {num_potential_files_in_post} files (≥{self.sfp_threshold}). Activating Subfolder per Post via [sfp] command.")
|
||||
should_create_post_subfolder = True
|
||||
|
||||
base_folder_names_for_post_content = []
|
||||
determined_post_save_path_for_history = self.override_output_dir if self.override_output_dir else self.download_root
|
||||
if not self.extract_links_only and self.use_subfolders:
|
||||
@@ -2327,9 +2331,10 @@ class DownloadThread(QThread):
|
||||
manga_custom_filename_format="{published} {title}",
|
||||
manga_custom_date_format="YYYY-MM-DD" ,
|
||||
sfp_threshold=None,
|
||||
creator_name_cache=None
|
||||
|
||||
creator_name_cache=None,
|
||||
proxies=None
|
||||
):
|
||||
|
||||
super().__init__()
|
||||
self.api_url_input = api_url_input
|
||||
self.output_dir = output_dir
|
||||
@@ -2404,6 +2409,7 @@ class DownloadThread(QThread):
|
||||
self.domain_override = domain_override
|
||||
self.sfp_threshold = sfp_threshold
|
||||
self.creator_name_cache = creator_name_cache
|
||||
self.proxies = proxies
|
||||
|
||||
if self.compress_images and Image is None:
|
||||
self.logger("⚠️ Image compression disabled: Pillow library not found (DownloadThread).")
|
||||
@@ -2437,6 +2443,7 @@ class DownloadThread(QThread):
|
||||
|
||||
self.logger(" Starting post fetch (single-threaded download process)...")
|
||||
|
||||
# --- FIX: Removed duplicate proxies argument here ---
|
||||
post_generator = download_from_api(
|
||||
self.api_url_input,
|
||||
logger=self.logger,
|
||||
@@ -2451,9 +2458,11 @@ class DownloadThread(QThread):
|
||||
app_base_dir=self.app_base_dir,
|
||||
manga_filename_style_for_sort_check=self.manga_filename_style if self.manga_mode_active else None,
|
||||
processed_post_ids=self.processed_post_ids_set,
|
||||
fetch_all_first=self.fetch_first
|
||||
fetch_all_first=self.fetch_first,
|
||||
proxies=self.proxies
|
||||
)
|
||||
|
||||
processed_count_for_delay = 0
|
||||
for posts_batch_data in post_generator:
|
||||
if self.isInterruptionRequested():
|
||||
was_process_cancelled = True
|
||||
@@ -2464,6 +2473,11 @@ class DownloadThread(QThread):
|
||||
was_process_cancelled = True
|
||||
break
|
||||
|
||||
processed_count_for_delay += 1
|
||||
if processed_count_for_delay > 0 and processed_count_for_delay % 50 == 0:
|
||||
self.logger(" ⏳ Safety Pause: Waiting 10 seconds to respect server rate limits...")
|
||||
time.sleep(10)
|
||||
|
||||
worker_args = {
|
||||
'post_data': individual_post_data,
|
||||
'emitter': worker_signals_obj,
|
||||
@@ -2532,7 +2546,8 @@ class DownloadThread(QThread):
|
||||
'archive_only_mode': self.archive_only_mode,
|
||||
'manga_custom_filename_format': self.manga_custom_filename_format,
|
||||
'manga_custom_date_format': self.manga_custom_date_format,
|
||||
'sfp_threshold': self.sfp_threshold
|
||||
'sfp_threshold': self.sfp_threshold,
|
||||
'proxies': self.proxies
|
||||
}
|
||||
|
||||
post_processing_worker = PostProcessorWorker(**worker_args)
|
||||
|
||||
@@ -19,12 +19,14 @@ class AllcomicDownloadThread(QThread):
|
||||
finished_signal = pyqtSignal(int, int, bool)
|
||||
overall_progress_signal = pyqtSignal(int, int)
|
||||
|
||||
def __init__(self, url, output_dir, parent=None):
|
||||
# 1. Update __init__ to accept proxies
|
||||
def __init__(self, url, output_dir, parent=None, proxies=None):
|
||||
super().__init__(parent)
|
||||
self.comic_url = url
|
||||
self.output_dir = output_dir
|
||||
self.is_cancelled = False
|
||||
self.pause_event = parent.pause_event if hasattr(parent, 'pause_event') else threading.Event()
|
||||
self.proxies = proxies # Store the proxies
|
||||
|
||||
def _check_pause(self):
|
||||
if self.is_cancelled: return True
|
||||
@@ -40,13 +42,19 @@ class AllcomicDownloadThread(QThread):
|
||||
grand_total_dl = 0
|
||||
grand_total_skip = 0
|
||||
|
||||
# Create the scraper session ONCE for the entire job
|
||||
scraper = cloudscraper.create_scraper(
|
||||
browser={'browser': 'firefox', 'platform': 'windows', 'desktop': True}
|
||||
)
|
||||
if self.proxies:
|
||||
self.progress_signal.emit(f" 🌍 Network: Using Proxy {self.proxies}")
|
||||
else:
|
||||
self.progress_signal.emit(" 🌍 Network: Direct Connection (No Proxy)")
|
||||
|
||||
# Pass the scraper to the function
|
||||
chapters_to_download = allcomic_get_list(scraper, self.comic_url, self.progress_signal.emit)
|
||||
scraper = requests.Session()
|
||||
scraper.headers.update({
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
|
||||
})
|
||||
|
||||
# 2. Pass self.proxies to get_chapter_list
|
||||
chapters_to_download = allcomic_get_list(scraper, self.comic_url, self.progress_signal.emit, proxies=self.proxies)
|
||||
|
||||
if not chapters_to_download:
|
||||
chapters_to_download = [self.comic_url]
|
||||
@@ -57,8 +65,9 @@ class AllcomicDownloadThread(QThread):
|
||||
if self._check_pause(): break
|
||||
|
||||
self.progress_signal.emit(f"\n-- Processing Chapter {chapter_idx + 1}/{len(chapters_to_download)} --")
|
||||
# Pass the scraper to the function
|
||||
comic_title, chapter_title, image_urls = allcomic_fetch_data(scraper, chapter_url, self.progress_signal.emit)
|
||||
|
||||
# 3. Pass self.proxies to fetch_chapter_data
|
||||
comic_title, chapter_title, image_urls = allcomic_fetch_data(scraper, chapter_url, self.progress_signal.emit, proxies=self.proxies)
|
||||
|
||||
if not image_urls:
|
||||
self.progress_signal.emit(f"❌ Failed to get data for chapter. Skipping.")
|
||||
@@ -80,6 +89,9 @@ class AllcomicDownloadThread(QThread):
|
||||
self.overall_progress_signal.emit(total_files_in_chapter, 0)
|
||||
headers = {'Referer': chapter_url}
|
||||
|
||||
# 4. Define smart timeout for images
|
||||
img_timeout = (30, 120) if self.proxies else 60
|
||||
|
||||
for i, img_url in enumerate(image_urls):
|
||||
if self._check_pause(): break
|
||||
|
||||
@@ -97,8 +109,9 @@ class AllcomicDownloadThread(QThread):
|
||||
if self._check_pause(): break
|
||||
try:
|
||||
self.progress_signal.emit(f" Downloading ({i+1}/{total_files_in_chapter}): '{filename}' (Attempt {attempt + 1})...")
|
||||
# Use the persistent scraper object
|
||||
response = scraper.get(img_url, stream=True, headers=headers, timeout=60)
|
||||
|
||||
# 5. Use proxies, verify=False, and new timeout
|
||||
response = scraper.get(img_url, stream=True, headers=headers, timeout=img_timeout, proxies=self.proxies, verify=False)
|
||||
response.raise_for_status()
|
||||
|
||||
with open(filepath, 'wb') as f:
|
||||
@@ -125,7 +138,7 @@ class AllcomicDownloadThread(QThread):
|
||||
grand_total_skip += 1
|
||||
|
||||
self.overall_progress_signal.emit(total_files_in_chapter, i + 1)
|
||||
time.sleep(0.5) # Increased delay between images for this site
|
||||
time.sleep(0.5)
|
||||
|
||||
if self._check_pause(): break
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@ import os
|
||||
import time
|
||||
import requests
|
||||
import re
|
||||
import random # Needed for random delays
|
||||
from datetime import datetime
|
||||
from concurrent.futures import ThreadPoolExecutor, wait
|
||||
from PyQt5.QtCore import QThread, pyqtSignal
|
||||
from ...core.deviantart_client import DeviantArtClient
|
||||
from ...utils.file_utils import clean_folder_name
|
||||
@@ -14,28 +14,29 @@ class DeviantArtDownloadThread(QThread):
|
||||
overall_progress_signal = pyqtSignal(int, int)
|
||||
finished_signal = pyqtSignal(int, int, bool, list)
|
||||
|
||||
def __init__(self, url, output_dir, pause_event, cancellation_event, parent=None):
|
||||
# 1. Accept proxies in init
|
||||
def __init__(self, url, output_dir, pause_event, cancellation_event, parent=None, proxies=None):
|
||||
super().__init__(parent)
|
||||
self.url = url
|
||||
self.output_dir = output_dir
|
||||
self.pause_event = pause_event
|
||||
self.cancellation_event = cancellation_event
|
||||
|
||||
# --- PASS LOGGER TO CLIENT ---
|
||||
# This ensures client logs go to the UI, not just the black console window
|
||||
self.client = DeviantArtClient(logger_func=self.progress_signal.emit)
|
||||
|
||||
self.proxies = proxies # Store proxies
|
||||
|
||||
self.parent_app = parent
|
||||
self.download_count = 0
|
||||
self.skip_count = 0
|
||||
|
||||
# --- THREAD SETTINGS ---
|
||||
self.max_threads = 10
|
||||
|
||||
def run(self):
|
||||
self.client = DeviantArtClient(logger_func=self.progress_signal.emit, proxies=self.proxies)
|
||||
|
||||
if self.proxies:
|
||||
self.progress_signal.emit(f" 🌍 Network: Using Proxy {self.proxies}")
|
||||
else:
|
||||
self.progress_signal.emit(" 🌍 Network: Direct Connection")
|
||||
|
||||
self.progress_signal.emit("=" * 40)
|
||||
self.progress_signal.emit(f"🚀 Starting DeviantArt download for: {self.url}")
|
||||
self.progress_signal.emit(f" ℹ️ Using {self.max_threads} parallel threads.")
|
||||
|
||||
try:
|
||||
if not self.client.authenticate():
|
||||
@@ -91,26 +92,25 @@ class DeviantArtDownloadThread(QThread):
|
||||
if not os.path.exists(base_folder):
|
||||
os.makedirs(base_folder, exist_ok=True)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=self.max_threads) as executor:
|
||||
while has_more:
|
||||
while has_more:
|
||||
if self._check_pause_cancel(): break
|
||||
|
||||
data = self.client.get_gallery_folder(username, offset=offset)
|
||||
results = data.get('results', [])
|
||||
has_more = data.get('has_more', False)
|
||||
offset = data.get('next_offset')
|
||||
|
||||
if not results: break
|
||||
|
||||
for deviation in results:
|
||||
if self._check_pause_cancel(): break
|
||||
|
||||
data = self.client.get_gallery_folder(username, offset=offset)
|
||||
results = data.get('results', [])
|
||||
has_more = data.get('has_more', False)
|
||||
offset = data.get('next_offset')
|
||||
|
||||
if not results: break
|
||||
|
||||
futures = []
|
||||
for deviation in results:
|
||||
if self._check_pause_cancel(): break
|
||||
future = executor.submit(self._process_deviation_task, deviation, base_folder)
|
||||
futures.append(future)
|
||||
|
||||
wait(futures)
|
||||
self._process_deviation_task(deviation, base_folder)
|
||||
|
||||
time.sleep(1)
|
||||
# 4. FIX 429: Add a small random delay between items
|
||||
# This prevents hammering the API 24 times in a single second.
|
||||
time.sleep(random.uniform(0.5, 1.2))
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
def _process_deviation_task(self, deviation, base_folder):
|
||||
if self._check_pause_cancel(): return
|
||||
@@ -173,7 +173,7 @@ class DeviantArtDownloadThread(QThread):
|
||||
final_filename = f"{clean_folder_name(new_name)}{ext}"
|
||||
|
||||
except Exception as e:
|
||||
self.progress_signal.emit(f" ⚠️ Renaming failed ({e}), using default.")
|
||||
pass
|
||||
|
||||
save_dir = override_dir if override_dir else self.output_dir
|
||||
if not os.path.exists(save_dir):
|
||||
@@ -189,7 +189,11 @@ class DeviantArtDownloadThread(QThread):
|
||||
try:
|
||||
self.progress_signal.emit(f" ⬇️ Downloading: {final_filename}")
|
||||
|
||||
with requests.get(file_url, stream=True, timeout=30) as r:
|
||||
# 5. Determine smart timeout for files
|
||||
timeout_val = (30, 120) if self.proxies else 30
|
||||
|
||||
# 6. Use proxies and verify=False
|
||||
with requests.get(file_url, stream=True, timeout=timeout_val, proxies=self.proxies, verify=False) as r:
|
||||
r.raise_for_status()
|
||||
|
||||
with open(filepath, 'wb') as f:
|
||||
|
||||
@@ -25,6 +25,7 @@ from .saint2_downloader_thread import Saint2DownloadThread
|
||||
from .simp_city_downloader_thread import SimpCityDownloadThread
|
||||
from .toonily_downloader_thread import ToonilyDownloadThread
|
||||
from .deviantart_downloader_thread import DeviantArtDownloadThread
|
||||
from .hentaifox_downloader_thread import HentaiFoxDownloadThread
|
||||
|
||||
def create_downloader_thread(main_app, api_url, service, id1, id2, effective_output_dir_for_run):
|
||||
"""
|
||||
@@ -185,6 +186,17 @@ def create_downloader_thread(main_app, api_url, service, id1, id2, effective_out
|
||||
cancellation_event=main_app.cancellation_event,
|
||||
parent=main_app
|
||||
)
|
||||
|
||||
# Handler for HentaiFox (New)
|
||||
if 'hentaifox.com' in api_url or service == 'hentaifox':
|
||||
main_app.log_signal.emit("🦊 HentaiFox URL detected.")
|
||||
return HentaiFoxDownloadThread(
|
||||
url_or_id=api_url,
|
||||
output_dir=effective_output_dir_for_run,
|
||||
parent=main_app
|
||||
)
|
||||
|
||||
|
||||
# ----------------------
|
||||
# --- Fallback ---
|
||||
# If no specific handler matched based on service name or URL pattern, return None.
|
||||
|
||||
136
src/ui/classes/hentaifox_downloader_thread.py
Normal file
136
src/ui/classes/hentaifox_downloader_thread.py
Normal file
@@ -0,0 +1,136 @@
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
from PyQt5.QtCore import QThread, pyqtSignal
|
||||
from ...core.hentaifox_client import get_gallery_metadata, get_image_link_for_page, get_gallery_id
|
||||
from ...utils.file_utils import clean_folder_name
|
||||
|
||||
class HentaiFoxDownloadThread(QThread):
|
||||
progress_signal = pyqtSignal(str) # Log messages
|
||||
file_progress_signal = pyqtSignal(str, object) # filename, (current_bytes, total_bytes)
|
||||
# finished_signal: (downloaded_count, skipped_count, was_cancelled, kept_files_list)
|
||||
finished_signal = pyqtSignal(int, int, bool, list)
|
||||
|
||||
def __init__(self, url_or_id, output_dir, parent=None):
|
||||
super().__init__(parent)
|
||||
self.gallery_id = get_gallery_id(url_or_id)
|
||||
self.output_dir = output_dir
|
||||
self.is_running = True
|
||||
self.downloaded_count = 0
|
||||
self.skipped_count = 0
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
self.progress_signal.emit(f"🔍 [HentaiFox] Fetching metadata for ID: {self.gallery_id}...")
|
||||
|
||||
# 1. Get Info
|
||||
try:
|
||||
data = get_gallery_metadata(self.gallery_id)
|
||||
except Exception as e:
|
||||
self.progress_signal.emit(f"❌ [HentaiFox] Failed to fetch metadata: {e}")
|
||||
self.finished_signal.emit(0, 0, False, [])
|
||||
return
|
||||
|
||||
title = clean_folder_name(data['title'])
|
||||
total_pages = data['total_pages']
|
||||
|
||||
# 2. Setup Folder
|
||||
save_folder = os.path.join(self.output_dir, f"[{self.gallery_id}] {title}")
|
||||
os.makedirs(save_folder, exist_ok=True)
|
||||
|
||||
self.progress_signal.emit(f"📂 Saving to: {save_folder}")
|
||||
self.progress_signal.emit(f"📄 Found {total_pages} pages. Starting download...")
|
||||
|
||||
# 3. Iterate and Download
|
||||
for i in range(1, total_pages + 1):
|
||||
if not self.is_running:
|
||||
self.progress_signal.emit("🛑 Download cancelled by user.")
|
||||
break
|
||||
|
||||
# Fetch image link for this specific page
|
||||
try:
|
||||
img_url = get_image_link_for_page(self.gallery_id, i)
|
||||
|
||||
if img_url:
|
||||
ext = img_url.split('.')[-1]
|
||||
filename = f"{i:03d}.{ext}"
|
||||
filepath = os.path.join(save_folder, filename)
|
||||
|
||||
# Check if exists
|
||||
if os.path.exists(filepath):
|
||||
self.progress_signal.emit(f"⚠️ [{i}/{total_pages}] Skipped (Exists): {filename}")
|
||||
self.skipped_count += 1
|
||||
else:
|
||||
self.progress_signal.emit(f"⬇️ [{i}/{total_pages}] Downloading: {filename}")
|
||||
|
||||
# CALL NEW DOWNLOAD FUNCTION
|
||||
success = self.download_image_with_progress(img_url, filepath, filename)
|
||||
|
||||
if success:
|
||||
self.progress_signal.emit(f"✅ [{i}/{total_pages}] Finished: {filename}")
|
||||
self.downloaded_count += 1
|
||||
else:
|
||||
self.progress_signal.emit(f"❌ [{i}/{total_pages}] Failed: {filename}")
|
||||
self.skipped_count += 1
|
||||
else:
|
||||
self.progress_signal.emit(f"❌ [{i}/{total_pages}] Error: No image link found.")
|
||||
self.skipped_count += 1
|
||||
|
||||
except Exception as e:
|
||||
self.progress_signal.emit(f"❌ [{i}/{total_pages}] Exception: {e}")
|
||||
self.skipped_count += 1
|
||||
|
||||
time.sleep(0.5)
|
||||
|
||||
# 4. Final Summary
|
||||
summary = (
|
||||
f"\n🏁 [HentaiFox] Task Complete!\n"
|
||||
f" - Total: {total_pages}\n"
|
||||
f" - Downloaded: {self.downloaded_count}\n"
|
||||
f" - Skipped: {self.skipped_count}\n"
|
||||
)
|
||||
self.progress_signal.emit(summary)
|
||||
|
||||
except Exception as e:
|
||||
self.progress_signal.emit(f"❌ Critical Error: {str(e)}")
|
||||
|
||||
self.finished_signal.emit(self.downloaded_count, self.skipped_count, not self.is_running, [])
|
||||
|
||||
def download_image_with_progress(self, url, path, filename):
|
||||
"""Downloads file while emitting byte-level progress signals."""
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
|
||||
"Referer": "https://hentaifox.com/"
|
||||
}
|
||||
|
||||
try:
|
||||
# stream=True is required to get size before downloading body
|
||||
r = requests.get(url, headers=headers, stream=True, timeout=20)
|
||||
if r.status_code != 200:
|
||||
return False
|
||||
|
||||
# Get Total Size (in bytes)
|
||||
total_size = int(r.headers.get('content-length', 0))
|
||||
downloaded_size = 0
|
||||
|
||||
chunk_size = 1024 # 1KB chunks
|
||||
|
||||
with open(path, 'wb') as f:
|
||||
for chunk in r.iter_content(chunk_size):
|
||||
if not self.is_running:
|
||||
r.close()
|
||||
return False
|
||||
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
downloaded_size += len(chunk)
|
||||
|
||||
self.file_progress_signal.emit(filename, (downloaded_size, total_size))
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Download Error: {e}")
|
||||
return False
|
||||
|
||||
def stop(self):
|
||||
self.is_running = False
|
||||
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
import time
|
||||
import cloudscraper
|
||||
import requests
|
||||
from PyQt5.QtCore import QThread, pyqtSignal
|
||||
|
||||
from ...utils.file_utils import clean_folder_name
|
||||
@@ -17,68 +17,78 @@ class NhentaiDownloadThread(QThread):
|
||||
|
||||
EXTENSION_MAP = {'j': 'jpg', 'p': 'png', 'g': 'gif', 'w': 'webp' }
|
||||
|
||||
# 1. Update init to initialize self.proxies
|
||||
def __init__(self, gallery_data, output_dir, parent=None):
|
||||
super().__init__(parent)
|
||||
self.gallery_data = gallery_data
|
||||
self.output_dir = output_dir
|
||||
self.is_cancelled = False
|
||||
self.proxies = None # Placeholder, will be injected by main_window
|
||||
|
||||
def run(self):
|
||||
# 2. Log Proxy Usage
|
||||
if self.proxies:
|
||||
self.progress_signal.emit(f" 🌍 Network: Using Proxy {self.proxies}")
|
||||
else:
|
||||
self.progress_signal.emit(" 🌍 Network: Direct Connection (No Proxy)")
|
||||
|
||||
title = self.gallery_data.get("title", {}).get("english", f"gallery_{self.gallery_data.get('id')}")
|
||||
gallery_id = self.gallery_data.get("id")
|
||||
media_id = self.gallery_data.get("media_id")
|
||||
pages_info = self.gallery_data.get("pages", [])
|
||||
|
||||
folder_name = clean_folder_name(title)
|
||||
gallery_path = os.path.join(self.output_dir, folder_name)
|
||||
|
||||
save_path = os.path.join(self.output_dir, folder_name)
|
||||
|
||||
try:
|
||||
os.makedirs(gallery_path, exist_ok=True)
|
||||
except OSError as e:
|
||||
self.progress_signal.emit(f"❌ Critical error creating directory: {e}")
|
||||
os.makedirs(save_path, exist_ok=True)
|
||||
self.progress_signal.emit(f" Saving to: {folder_name}")
|
||||
except Exception as e:
|
||||
self.progress_signal.emit(f" ❌ Error creating directory: {e}")
|
||||
self.finished_signal.emit(0, len(pages_info), False)
|
||||
return
|
||||
|
||||
self.progress_signal.emit(f"⬇️ Downloading '{title}' to folder '{folder_name}'...")
|
||||
|
||||
scraper = cloudscraper.create_scraper()
|
||||
download_count = 0
|
||||
skip_count = 0
|
||||
total_pages = len(pages_info)
|
||||
|
||||
# 3. Use requests.Session instead of cloudscraper
|
||||
scraper = requests.Session()
|
||||
|
||||
# 4. Smart timeout logic
|
||||
img_timeout = (30, 120) if self.proxies else 60
|
||||
|
||||
for i, page_data in enumerate(pages_info):
|
||||
if self.is_cancelled:
|
||||
break
|
||||
|
||||
page_num = i + 1
|
||||
if self.is_cancelled: break
|
||||
|
||||
ext_char = page_data.get('t', 'j')
|
||||
extension = self.EXTENSION_MAP.get(ext_char, 'jpg')
|
||||
|
||||
relative_path = f"/galleries/{media_id}/{page_num}.{extension}"
|
||||
|
||||
local_filename = f"{page_num:03d}.{extension}"
|
||||
filepath = os.path.join(gallery_path, local_filename)
|
||||
file_ext = self.EXTENSION_MAP.get(page_data.get('t'), 'jpg')
|
||||
local_filename = f"{i+1:03d}.{file_ext}"
|
||||
filepath = os.path.join(save_path, local_filename)
|
||||
|
||||
if os.path.exists(filepath):
|
||||
self.progress_signal.emit(f" -> Skip (Exists): {local_filename}")
|
||||
self.progress_signal.emit(f" Skipping {local_filename} (already exists).")
|
||||
skip_count += 1
|
||||
continue
|
||||
|
||||
download_successful = False
|
||||
|
||||
# Try servers until one works
|
||||
for server in self.IMAGE_SERVERS:
|
||||
if self.is_cancelled:
|
||||
break
|
||||
if self.is_cancelled: break
|
||||
|
||||
# Construct URL: server/galleries/media_id/page_num.ext
|
||||
full_url = f"{server}/galleries/{media_id}/{i+1}.{file_ext}"
|
||||
|
||||
full_url = f"{server}{relative_path}"
|
||||
try:
|
||||
self.progress_signal.emit(f" Downloading page {page_num}/{len(pages_info)} from {server} ...")
|
||||
self.progress_signal.emit(f" Downloading page {i+1}/{total_pages}...")
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Referer': f'https://nhentai.net/g/{gallery_id}/'
|
||||
}
|
||||
|
||||
response = scraper.get(full_url, headers=headers, timeout=60, stream=True)
|
||||
# 5. Add proxies, verify=False, and timeout
|
||||
response = scraper.get(full_url, headers=headers, timeout=img_timeout, stream=True, proxies=self.proxies, verify=False)
|
||||
|
||||
if response.status_code == 200:
|
||||
with open(filepath, 'wb') as f:
|
||||
@@ -86,12 +96,14 @@ class NhentaiDownloadThread(QThread):
|
||||
f.write(chunk)
|
||||
download_count += 1
|
||||
download_successful = True
|
||||
break
|
||||
break # Stop trying servers
|
||||
else:
|
||||
self.progress_signal.emit(f" -> {server} returned status {response.status_code}. Trying next server...")
|
||||
# self.progress_signal.emit(f" -> {server} returned status {response.status_code}...")
|
||||
pass
|
||||
|
||||
except Exception as e:
|
||||
self.progress_signal.emit(f" -> {server} failed to connect or timed out: {e}. Trying next server...")
|
||||
# self.progress_signal.emit(f" -> {server} failed: {e}")
|
||||
pass
|
||||
|
||||
if not download_successful:
|
||||
self.progress_signal.emit(f" ❌ Failed to download {local_filename} from all servers.")
|
||||
|
||||
@@ -5,10 +5,11 @@ import sys
|
||||
|
||||
# --- PyQt5 Imports ---
|
||||
from PyQt5.QtCore import Qt, QStandardPaths, QTimer
|
||||
from PyQt5.QtGui import QIntValidator # <--- NEW: Added for Port validation
|
||||
from PyQt5.QtWidgets import (
|
||||
QApplication, QDialog, QHBoxLayout, QLabel, QPushButton, QVBoxLayout,
|
||||
QGroupBox, QComboBox, QMessageBox, QGridLayout, QCheckBox, QLineEdit,
|
||||
QTabWidget, QWidget, QFileDialog # Added QFileDialog
|
||||
QTabWidget, QWidget, QFileDialog
|
||||
)
|
||||
# --- Local Application Imports ---
|
||||
from ...i18n.translator import get_translation
|
||||
@@ -21,7 +22,9 @@ from ...config.constants import (
|
||||
RESOLUTION_KEY, UI_SCALE_KEY, SAVE_CREATOR_JSON_KEY,
|
||||
DATE_PREFIX_FORMAT_KEY,
|
||||
COOKIE_TEXT_KEY, USE_COOKIE_KEY,
|
||||
FETCH_FIRST_KEY, DISCORD_TOKEN_KEY, POST_DOWNLOAD_ACTION_KEY
|
||||
FETCH_FIRST_KEY, DISCORD_TOKEN_KEY, POST_DOWNLOAD_ACTION_KEY,
|
||||
PROXY_ENABLED_KEY, PROXY_HOST_KEY, PROXY_PORT_KEY,
|
||||
PROXY_USERNAME_KEY, PROXY_PASSWORD_KEY
|
||||
)
|
||||
from ...services.updater import UpdateChecker, UpdateDownloader
|
||||
|
||||
@@ -118,16 +121,15 @@ class FutureSettingsDialog(QDialog):
|
||||
super().__init__(parent)
|
||||
self.parent_app = parent_app_ref
|
||||
self.setModal(True)
|
||||
self.update_downloader_thread = None # To keep a reference
|
||||
self.update_downloader_thread = None
|
||||
|
||||
app_icon = get_app_icon_object()
|
||||
if app_icon and not app_icon.isNull():
|
||||
self.setWindowIcon(app_icon)
|
||||
|
||||
screen_height = QApplication.primaryScreen().availableGeometry().height() if QApplication.primaryScreen() else 800
|
||||
# Use a more balanced aspect ratio
|
||||
scale_factor = screen_height / 1000.0
|
||||
base_min_w, base_min_h = 480, 420 # Wider, less tall
|
||||
base_min_w, base_min_h = 550, 450 # <--- TWEAK: Slightly increased width for better layout
|
||||
scaled_min_w = int(base_min_w * scale_factor)
|
||||
scaled_min_h = int(base_min_h * scale_factor)
|
||||
self.setMinimumSize(scaled_min_w, scaled_min_h)
|
||||
@@ -135,6 +137,9 @@ class FutureSettingsDialog(QDialog):
|
||||
self._init_ui()
|
||||
self._retranslate_ui()
|
||||
self._apply_theme()
|
||||
|
||||
# <--- NEW: Load proxy settings on init
|
||||
self._load_proxy_settings()
|
||||
|
||||
def _init_ui(self):
|
||||
"""Initializes all UI components and layouts for the dialog."""
|
||||
@@ -147,14 +152,16 @@ class FutureSettingsDialog(QDialog):
|
||||
# --- Create Tabs ---
|
||||
self.display_tab = QWidget()
|
||||
self.downloads_tab = QWidget()
|
||||
self.network_tab = QWidget() # <--- NEW: Network Tab
|
||||
self.updates_tab = QWidget()
|
||||
|
||||
# Add tabs to the widget
|
||||
self.tab_widget.addTab(self.display_tab, "Display")
|
||||
self.tab_widget.addTab(self.downloads_tab, "Downloads")
|
||||
self.tab_widget.addTab(self.network_tab, "Proxy/Network") # <--- NEW
|
||||
self.tab_widget.addTab(self.updates_tab, "Updates")
|
||||
|
||||
# --- Populate Display Tab ---
|
||||
# [Display Tab Code (Unchanged) ...]
|
||||
display_tab_layout = QVBoxLayout(self.display_tab)
|
||||
self.display_group_box = QGroupBox()
|
||||
display_layout = QGridLayout(self.display_group_box)
|
||||
@@ -184,9 +191,9 @@ class FutureSettingsDialog(QDialog):
|
||||
display_layout.addWidget(self.resolution_combo_box, 3, 1)
|
||||
|
||||
display_tab_layout.addWidget(self.display_group_box)
|
||||
display_tab_layout.addStretch(1) # Push content to the top
|
||||
display_tab_layout.addStretch(1)
|
||||
|
||||
# --- Populate Downloads Tab ---
|
||||
# [Downloads Tab Code (Unchanged) ...]
|
||||
downloads_tab_layout = QVBoxLayout(self.downloads_tab)
|
||||
self.download_settings_group_box = QGroupBox()
|
||||
download_settings_layout = QGridLayout(self.download_settings_group_box)
|
||||
@@ -217,7 +224,6 @@ class FutureSettingsDialog(QDialog):
|
||||
self.fetch_first_checkbox.stateChanged.connect(self._fetch_first_setting_changed)
|
||||
download_settings_layout.addWidget(self.fetch_first_checkbox, 4, 0, 1, 2)
|
||||
|
||||
# --- START: Add new Load/Save buttons ---
|
||||
settings_file_layout = QHBoxLayout()
|
||||
self.load_settings_button = QPushButton()
|
||||
self.save_settings_button = QPushButton()
|
||||
@@ -225,18 +231,72 @@ class FutureSettingsDialog(QDialog):
|
||||
settings_file_layout.addWidget(self.save_settings_button)
|
||||
settings_file_layout.addStretch(1)
|
||||
|
||||
# Add this new layout to the grid
|
||||
download_settings_layout.addLayout(settings_file_layout, 5, 0, 1, 2) # Row 5, span 2 cols
|
||||
download_settings_layout.addLayout(settings_file_layout, 5, 0, 1, 2)
|
||||
|
||||
# Connect signals
|
||||
self.load_settings_button.clicked.connect(self._handle_load_settings)
|
||||
self.save_settings_button.clicked.connect(self._handle_save_settings)
|
||||
# --- END: Add new Load/Save buttons ---
|
||||
|
||||
downloads_tab_layout.addWidget(self.download_settings_group_box)
|
||||
downloads_tab_layout.addStretch(1) # Push content to the top
|
||||
downloads_tab_layout.addStretch(1)
|
||||
|
||||
# --- Populate Updates Tab ---
|
||||
# --- START: Network Tab (NEW) ---
|
||||
network_tab_layout = QVBoxLayout(self.network_tab)
|
||||
self.proxy_group_box = QGroupBox()
|
||||
proxy_layout = QGridLayout(self.proxy_group_box)
|
||||
|
||||
# Enable Checkbox
|
||||
self.proxy_enabled_checkbox = QCheckBox()
|
||||
self.proxy_enabled_checkbox.stateChanged.connect(self._proxy_setting_changed)
|
||||
proxy_layout.addWidget(self.proxy_enabled_checkbox, 0, 0, 1, 2)
|
||||
|
||||
# Proxy Type Dropdown
|
||||
self.proxy_type_label = QLabel("Proxy Type:")
|
||||
self.proxy_type_combo = QComboBox()
|
||||
self.proxy_type_combo.addItems(["HTTP", "SOCKS4", "SOCKS5"])
|
||||
self.proxy_type_combo.currentIndexChanged.connect(self._proxy_setting_changed)
|
||||
proxy_layout.addWidget(self.proxy_type_label, 1, 0)
|
||||
proxy_layout.addWidget(self.proxy_type_combo, 1, 1)
|
||||
|
||||
|
||||
# Host / IP
|
||||
self.proxy_host_label = QLabel()
|
||||
self.proxy_host_input = QLineEdit()
|
||||
self.proxy_host_input.setPlaceholderText("127.0.0.1")
|
||||
self.proxy_host_input.editingFinished.connect(self._proxy_setting_changed)
|
||||
proxy_layout.addWidget(self.proxy_host_label, 2, 0) # Changed row to 2
|
||||
proxy_layout.addWidget(self.proxy_host_input, 2, 1)
|
||||
|
||||
# Port
|
||||
self.proxy_port_label = QLabel()
|
||||
self.proxy_port_input = QLineEdit()
|
||||
self.proxy_port_input.setPlaceholderText("8080")
|
||||
self.proxy_port_input.setValidator(QIntValidator(1, 65535, self)) # Only numbers
|
||||
self.proxy_port_input.editingFinished.connect(self._proxy_setting_changed)
|
||||
proxy_layout.addWidget(self.proxy_port_label, 3, 0)
|
||||
proxy_layout.addWidget(self.proxy_port_input, 3, 1)
|
||||
|
||||
# Username
|
||||
self.proxy_user_label = QLabel()
|
||||
self.proxy_user_input = QLineEdit()
|
||||
self.proxy_user_input.setPlaceholderText("(Optional)")
|
||||
self.proxy_user_input.editingFinished.connect(self._proxy_setting_changed)
|
||||
proxy_layout.addWidget(self.proxy_user_label, 4, 0)
|
||||
proxy_layout.addWidget(self.proxy_user_input, 4, 1)
|
||||
|
||||
# Password
|
||||
self.proxy_pass_label = QLabel()
|
||||
self.proxy_pass_input = QLineEdit()
|
||||
self.proxy_pass_input.setPlaceholderText("(Optional)")
|
||||
self.proxy_pass_input.setEchoMode(QLineEdit.Password) # Mask input
|
||||
self.proxy_pass_input.editingFinished.connect(self._proxy_setting_changed)
|
||||
proxy_layout.addWidget(self.proxy_pass_label, 5, 0)
|
||||
proxy_layout.addWidget(self.proxy_pass_input, 5, 1)
|
||||
|
||||
network_tab_layout.addWidget(self.proxy_group_box)
|
||||
network_tab_layout.addStretch(1)
|
||||
# --- END: Network Tab (NEW) ---
|
||||
|
||||
# [Updates Tab Code (Unchanged) ...]
|
||||
updates_tab_layout = QVBoxLayout(self.updates_tab)
|
||||
self.update_group_box = QGroupBox()
|
||||
update_layout = QGridLayout(self.update_group_box)
|
||||
@@ -249,7 +309,7 @@ class FutureSettingsDialog(QDialog):
|
||||
update_layout.addWidget(self.check_update_button, 1, 0, 1, 2)
|
||||
|
||||
updates_tab_layout.addWidget(self.update_group_box)
|
||||
updates_tab_layout.addStretch(1) # Push content to the top
|
||||
updates_tab_layout.addStretch(1)
|
||||
|
||||
# --- OK Button (outside tabs) ---
|
||||
button_layout = QHBoxLayout()
|
||||
@@ -266,16 +326,17 @@ class FutureSettingsDialog(QDialog):
|
||||
# --- Tab Titles ---
|
||||
self.tab_widget.setTabText(0, self._tr("settings_tab_display", "Display"))
|
||||
self.tab_widget.setTabText(1, self._tr("settings_tab_downloads", "Downloads"))
|
||||
self.tab_widget.setTabText(2, self._tr("settings_tab_updates", "Updates"))
|
||||
self.tab_widget.setTabText(2, self._tr("settings_tab_network", "Proxy/Network")) # <--- NEW
|
||||
self.tab_widget.setTabText(3, self._tr("settings_tab_updates", "Updates"))
|
||||
|
||||
# --- Display Tab ---
|
||||
# [Display Tab (Unchanged) ...]
|
||||
self.display_group_box.setTitle(self._tr("display_settings_group_title", "Display Settings"))
|
||||
self.theme_label.setText(self._tr("theme_label", "Theme:"))
|
||||
self.ui_scale_label.setText(self._tr("ui_scale_label", "UI Scale:"))
|
||||
self.language_label.setText(self._tr("language_label", "Language:"))
|
||||
self.window_size_label.setText(self._tr("window_size_label", "Window Size:"))
|
||||
|
||||
# --- Downloads Tab ---
|
||||
# [Downloads Tab (Unchanged) ...]
|
||||
self.download_settings_group_box.setTitle(self._tr("download_settings_group_title", "Download Settings"))
|
||||
self.default_path_label.setText(self._tr("default_path_label", "Default Path:"))
|
||||
self.date_prefix_format_label.setText(self._tr("date_prefix_format_label", "Post Subfolder Format:"))
|
||||
@@ -294,32 +355,112 @@ class FutureSettingsDialog(QDialog):
|
||||
self.fetch_first_checkbox.setToolTip(self._tr("fetch_first_tooltip", "If checked, the downloader will find all posts from a creator first before starting any downloads.\nThis can be slower to start but provides a more accurate progress bar."))
|
||||
self.save_path_button.setText(self._tr("settings_save_all_button", "Save Path + Cookie + Token"))
|
||||
self.save_path_button.setToolTip(self._tr("settings_save_all_tooltip", "Save the current 'Download Location', Cookie, and Discord Token settings for future sessions."))
|
||||
|
||||
# --- START: Add new button text ---
|
||||
self.load_settings_button.setText(self._tr("load_settings_button", "Load Settings..."))
|
||||
self.load_settings_button.setToolTip(self._tr("load_settings_tooltip", "Load all download settings from a .json file."))
|
||||
self.save_settings_button.setText(self._tr("save_settings_button", "Save Settings..."))
|
||||
self.save_settings_button.setToolTip(self._tr("save_settings_tooltip", "Save all current download settings to a .json file."))
|
||||
# --- END: Add new button text ---
|
||||
|
||||
# --- Updates Tab ---
|
||||
# --- START: Network Tab (NEW) ---
|
||||
self.proxy_group_box.setTitle(self._tr("proxy_settings_group_title", "Proxy Configuration"))
|
||||
self.proxy_enabled_checkbox.setText(self._tr("proxy_enabled_label", "Enable Proxy"))
|
||||
self.proxy_host_label.setText(self._tr("proxy_host_label", "Host / IP:"))
|
||||
self.proxy_port_label.setText(self._tr("proxy_port_label", "Port:"))
|
||||
self.proxy_user_label.setText(self._tr("proxy_user_label", "Username (Optional):"))
|
||||
self.proxy_pass_label.setText(self._tr("proxy_pass_label", "Password (Optional):"))
|
||||
# --- END: Network Tab (NEW) ---
|
||||
|
||||
# [Updates Tab (Unchanged) ...]
|
||||
self.update_group_box.setTitle(self._tr("update_group_title", "Application Updates"))
|
||||
current_version = self.parent_app.windowTitle().split(' v')[-1]
|
||||
self.version_label.setText(self._tr("current_version_label", f"Current Version: v{current_version}"))
|
||||
self.update_status_label.setText(self._tr("update_status_ready", "Ready to check."))
|
||||
self.check_update_button.setText(self._tr("check_for_updates_button", "Check for Updates"))
|
||||
|
||||
# --- General ---
|
||||
self._update_theme_toggle_button_text()
|
||||
self.ok_button.setText(self._tr("ok_button", "OK"))
|
||||
|
||||
# --- Load Data ---
|
||||
self._populate_display_combo_boxes()
|
||||
self._populate_language_combo_box()
|
||||
self._populate_post_download_action_combo()
|
||||
self._load_date_prefix_format()
|
||||
self._load_checkbox_states()
|
||||
|
||||
# --- START: New Proxy Logic ---
|
||||
def _load_proxy_settings(self):
|
||||
"""Loads proxy settings from QSettings into the UI."""
|
||||
# Block signals to prevent triggering auto-save while loading
|
||||
self.proxy_enabled_checkbox.blockSignals(True)
|
||||
self.proxy_type_combo.blockSignals(True) # <--- NEW
|
||||
self.proxy_host_input.blockSignals(True)
|
||||
self.proxy_port_input.blockSignals(True)
|
||||
self.proxy_user_input.blockSignals(True)
|
||||
self.proxy_pass_input.blockSignals(True)
|
||||
|
||||
# Load values
|
||||
enabled = self.parent_app.settings.value(PROXY_ENABLED_KEY, False, type=bool)
|
||||
proxy_type = self.parent_app.settings.value("proxy_type", "HTTP", type=str) # <--- NEW
|
||||
host = self.parent_app.settings.value(PROXY_HOST_KEY, "", type=str)
|
||||
port = self.parent_app.settings.value(PROXY_PORT_KEY, "", type=str)
|
||||
user = self.parent_app.settings.value(PROXY_USERNAME_KEY, "", type=str)
|
||||
password = self.parent_app.settings.value(PROXY_PASSWORD_KEY, "", type=str)
|
||||
|
||||
# Apply values to UI
|
||||
self.proxy_enabled_checkbox.setChecked(enabled)
|
||||
|
||||
# <--- NEW: Set the dropdown selection
|
||||
index = self.proxy_type_combo.findText(proxy_type)
|
||||
if index >= 0:
|
||||
self.proxy_type_combo.setCurrentIndex(index)
|
||||
else:
|
||||
self.proxy_type_combo.setCurrentIndex(0) # Default to first item if not found
|
||||
|
||||
self.proxy_host_input.setText(host)
|
||||
self.proxy_port_input.setText(port)
|
||||
self.proxy_user_input.setText(user)
|
||||
self.proxy_pass_input.setText(password)
|
||||
|
||||
self._update_proxy_fields_state(enabled)
|
||||
|
||||
# Unblock signals
|
||||
self.proxy_enabled_checkbox.blockSignals(False)
|
||||
self.proxy_type_combo.blockSignals(False) # <--- NEW
|
||||
self.proxy_host_input.blockSignals(False)
|
||||
self.proxy_port_input.blockSignals(False)
|
||||
self.proxy_user_input.blockSignals(False)
|
||||
self.proxy_pass_input.blockSignals(False)
|
||||
|
||||
def _proxy_setting_changed(self):
|
||||
"""Saves the current proxy UI state to QSettings."""
|
||||
enabled = self.proxy_enabled_checkbox.isChecked()
|
||||
proxy_type = self.proxy_type_combo.currentText() # <--- NEW
|
||||
host = self.proxy_host_input.text().strip()
|
||||
port = self.proxy_port_input.text().strip()
|
||||
user = self.proxy_user_input.text().strip()
|
||||
password = self.proxy_pass_input.text().strip()
|
||||
|
||||
self.parent_app.settings.setValue(PROXY_ENABLED_KEY, enabled)
|
||||
self.parent_app.settings.setValue("proxy_type", proxy_type) # <--- NEW
|
||||
self.parent_app.settings.setValue(PROXY_HOST_KEY, host)
|
||||
self.parent_app.settings.setValue(PROXY_PORT_KEY, port)
|
||||
self.parent_app.settings.setValue(PROXY_USERNAME_KEY, user)
|
||||
self.parent_app.settings.setValue(PROXY_PASSWORD_KEY, password)
|
||||
|
||||
self.parent_app.settings.sync()
|
||||
|
||||
self._update_proxy_fields_state(enabled)
|
||||
|
||||
# Optional: Notify main app that network settings changed if needed
|
||||
# self.parent_app.reload_proxy_settings()
|
||||
|
||||
def _update_proxy_fields_state(self, enabled):
|
||||
"""Enables or disables input fields based on the checkbox."""
|
||||
self.proxy_type_combo.setEnabled(enabled)
|
||||
self.proxy_host_input.setEnabled(enabled)
|
||||
self.proxy_port_input.setEnabled(enabled)
|
||||
self.proxy_user_input.setEnabled(enabled)
|
||||
self.proxy_pass_input.setEnabled(enabled)
|
||||
# --- END: New Proxy Logic ---
|
||||
|
||||
def _check_for_updates(self):
|
||||
self.check_update_button.setEnabled(False)
|
||||
self.update_status_label.setText(self._tr("update_status_checking", "Checking..."))
|
||||
|
||||
@@ -73,7 +73,6 @@ class HelpGuideDialog(QDialog):
|
||||
<li>fap-nation.org/</li>
|
||||
<li>Discord</li>
|
||||
<li>allporncomic.com</li>
|
||||
<li>allporncomic.com</li>
|
||||
<li>hentai2read.com</li>
|
||||
<li>mangadex.org</li>
|
||||
<li>Simpcity</li>
|
||||
@@ -279,6 +278,46 @@ class HelpGuideDialog(QDialog):
|
||||
</ul>
|
||||
"""),
|
||||
|
||||
("Add to Queue",
|
||||
"""
|
||||
<p>This feature allows you to queue up multiple distinct downloads with different settings and run them all sequentially.</p>
|
||||
|
||||
<h3 style='color: #E0E0E0;'>Step 1: Prepare the Download</h3>
|
||||
<p>Before clicking add, configure the download exactly how you want it processed for this specific link:</p>
|
||||
<ul>
|
||||
<li><b>Select Directory:</b> Choose where you want the files to go.</li>
|
||||
<li><b>Configure Options:</b> Check/uncheck boxes (e.g., "Separate Folders", "Use Cookie", "Manga Mode").</li>
|
||||
<li><b>Paste URL:</b> Enter the link for the creator or post you want to download.</li>
|
||||
</ul>
|
||||
|
||||
<h3 style='color: #E0E0E0;'>Step 2: Add to Queue</h3>
|
||||
<ol>
|
||||
<li>Click the <b>Add to Queue</b> button (located near the Start Download).</li>
|
||||
<li><b>Confirmation:</b> You will see a popup message and the log will print <code>✅ Job added to queue</code>.</li>
|
||||
<li>The URL box will clear, allowing you to immediately paste the next link.</li>
|
||||
</ol>
|
||||
|
||||
<h3 style='color: #E0E0E0;'>Step 3: Repeat & Start</h3>
|
||||
<p>You can repeat steps 1 and 2 as many times as you like. You can even change settings (like the download folder) between adds; the queue remembers the specific settings for each individual link.</p>
|
||||
<p>To start processing the queue:</p>
|
||||
<ol>
|
||||
<li>In the Link Input box, type exactly: <code>start queue</code></li>
|
||||
<li>The main "Start Download" button will change to <b>"🚀 Execute Queue"</b>.</li>
|
||||
<li>Click that button to begin.</li>
|
||||
</ol>
|
||||
|
||||
<h3 style='color: #E0E0E0;'>Processing Behavior</h3>
|
||||
<p>Once started, the app will lock the UI, load the first job, download it until finished, and automatically move to the next until the queue is empty.</p>
|
||||
|
||||
<h3 style='color: #E0E0E0;'>Special Case: Creator Selection Popup</h3>
|
||||
<p>If you use the <b>Creator Selection</b> popup (the 🎨 button):</p>
|
||||
<ul>
|
||||
<li>Select multiple creators in that popup and click <b>"Queue Selected"</b>.</li>
|
||||
<li>The app internally adds them to a temporary list.</li>
|
||||
<li>When you click the main <b>"Add to Queue"</b> button on the main window, it will detect that list and automatically bulk-create job files for all the creators you selected.</li>
|
||||
</ul>
|
||||
"""),
|
||||
|
||||
("Special Commands",
|
||||
"""
|
||||
<p>You can add special commands to the <b>"Filter by Character(s)"</b> input field to change download behavior for a single task. Commands are keywords wrapped in square brackets <code>[]</code>.</p>
|
||||
@@ -450,7 +489,16 @@ class HelpGuideDialog(QDialog):
|
||||
("Utility & Advanced Options",
|
||||
"""
|
||||
<p>These features provide advanced control over your downloads, sessions, and application settings.</p>
|
||||
|
||||
|
||||
<h3 style='color: #E0E0E0;'>🛡️ Proxy Support </h3>
|
||||
<p>You can now configure a proxy to bypass region blocks or ISP restrictions (e.g., for AllComic or Nhentai).</p>
|
||||
<p>Go to <b>Settings ⚙️ > Proxy Tab</b> to set it up:</p>
|
||||
<ul>
|
||||
<li><b>Protocols:</b> Full support for <b>HTTP</b>, <b>SOCKS4</b>, and <b>SOCKS5</b>.</li>
|
||||
<li><b>Authentication:</b> Supports username and password for private proxies.</li>
|
||||
<li><b>Global Effect:</b> Once enabled, all app connections (including API fetches and file downloads) will route through this proxy.</li>
|
||||
</ul>
|
||||
|
||||
<h3 style='color: #E0E0E0;'>Use Cookie</h3>
|
||||
<p>This is essential for downloading from sites that require a login (like <b>SimpCity</b> or accessing your <b>favorites</b> on Kemono/Coomer). You can either:</p>
|
||||
<ul>
|
||||
@@ -484,6 +532,7 @@ class HelpGuideDialog(QDialog):
|
||||
<li>Toggle <b>"Fetch First"</b> (to find all posts from a creator before starting any downloads).</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><b>Proxy Tab:</b> Configure HTTP/SOCKS proxies and authentication.</li>
|
||||
<li><b>Updates Tab:</b> Check for and install new application updates.</li>
|
||||
</ul>
|
||||
|
||||
@@ -605,7 +654,8 @@ class HelpGuideDialog(QDialog):
|
||||
main_layout.addLayout(content_layout, 1)
|
||||
|
||||
self.nav_list = QListWidget()
|
||||
self.nav_list.setFixedWidth(int(220 * scale))
|
||||
# Increased width to prevent scrollbar overlap
|
||||
self.nav_list.setFixedWidth(int(280 * scale))
|
||||
# Styles are now set in the __init__ method
|
||||
content_layout.addWidget(self.nav_list)
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
try:
|
||||
from fpdf import FPDF
|
||||
FPDF_AVAILABLE = True
|
||||
@@ -18,7 +20,9 @@ try:
|
||||
self.set_font(self.font_family_main, '', 8)
|
||||
self.cell(0, 10, 'Page ' + str(self.page_no()), 0, 0, 'C')
|
||||
|
||||
except ImportError:
|
||||
except Exception as e:
|
||||
print(f"\n❌ DEBUG INFO: Import failed. The specific error is: {e}")
|
||||
print(f"❌ DEBUG INFO: Python running this script is located at: {sys.executable}\n")
|
||||
FPDF_AVAILABLE = False
|
||||
FPDF = None
|
||||
PDF = None
|
||||
@@ -244,6 +248,9 @@ def create_single_pdf_from_content(posts_data, output_filename, font_path, add_i
|
||||
pdf.multi_cell(w=0, h=7, txt=post.get('content', 'No Content'))
|
||||
|
||||
try:
|
||||
output_dir = os.path.dirname(output_filename)
|
||||
if output_dir and not os.path.exists(output_dir):
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
pdf.output(output_filename)
|
||||
logger(f"✅ Successfully created single PDF: '{os.path.basename(output_filename)}'")
|
||||
return True
|
||||
|
||||
@@ -28,8 +28,8 @@ class UpdateCheckDialog(QDialog):
|
||||
self.selected_profiles_list = [] # Will store a list of {'name': ..., 'data': ...}
|
||||
|
||||
self._default_checkbox_tooltip = (
|
||||
"If checked, the settings from the selected profile will be loaded into the main window.\n"
|
||||
"You can then modify them. When you start the download, the new settings will be saved to the profile."
|
||||
"If checked, the settings fields will be unlocked and editable.\n"
|
||||
"If unchecked, settings will still load, but in 'Read-Only' mode."
|
||||
)
|
||||
|
||||
self._init_ui()
|
||||
@@ -65,13 +65,17 @@ class UpdateCheckDialog(QDialog):
|
||||
self.list_widget.itemChanged.connect(self._handle_item_changed)
|
||||
layout.addWidget(self.list_widget)
|
||||
|
||||
# --- NEW: Checkbox to Load Settings ---
|
||||
self.load_settings_checkbox = QCheckBox("Load profile settings into UI (Edit Settings)")
|
||||
self.load_settings_checkbox.setToolTip(self._default_checkbox_tooltip)
|
||||
layout.addWidget(self.load_settings_checkbox)
|
||||
# Renamed text to reflect new behavior
|
||||
self.edit_settings_checkbox = QCheckBox("Enable Editing (Unlock Settings)")
|
||||
self.edit_settings_checkbox.setToolTip(self._default_checkbox_tooltip)
|
||||
|
||||
# Checked by default as requested
|
||||
self.edit_settings_checkbox.setChecked(True)
|
||||
|
||||
layout.addWidget(self.edit_settings_checkbox)
|
||||
# -------------------------------------
|
||||
|
||||
# --- All Buttons in One Horizontal Layout ---
|
||||
|
||||
button_layout = QHBoxLayout()
|
||||
button_layout.setSpacing(6) # small even spacing between all buttons
|
||||
|
||||
@@ -110,7 +114,8 @@ class UpdateCheckDialog(QDialog):
|
||||
self.deselect_all_button.setText(self._tr("deselect_all_button_text", "Deselect All"))
|
||||
self.check_button.setText(self._tr("update_check_dialog_check_button", "Check Selected"))
|
||||
self.close_button.setText(self._tr("update_check_dialog_close_button", "Close"))
|
||||
self.load_settings_checkbox.setText(self._tr("update_check_load_settings_checkbox", "Load profile settings into UI (Edit Settings)"))
|
||||
# Updated translation key and default text
|
||||
self.edit_settings_checkbox.setText(self._tr("update_check_enable_editing_checkbox", "Enable Editing (Unlock Settings)"))
|
||||
|
||||
def _load_profiles(self):
|
||||
"""Loads all .json files from the creator_profiles directory as checkable items."""
|
||||
@@ -133,7 +138,6 @@ class UpdateCheckDialog(QDialog):
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Basic validation to ensure it's a valid profile
|
||||
if 'creator_url' in data and 'processed_post_ids' in data:
|
||||
creator_name = os.path.splitext(filename)[0]
|
||||
profiles_found.append({'name': creator_name, 'data': data})
|
||||
@@ -147,7 +151,6 @@ class UpdateCheckDialog(QDialog):
|
||||
for profile_info in profiles_found:
|
||||
item = QListWidgetItem(profile_info['name'])
|
||||
item.setData(Qt.UserRole, profile_info)
|
||||
# --- Make item checkable ---
|
||||
item.setFlags(item.flags() | Qt.ItemIsUserCheckable)
|
||||
item.setCheckState(Qt.Unchecked)
|
||||
self.list_widget.addItem(item)
|
||||
@@ -158,14 +161,13 @@ class UpdateCheckDialog(QDialog):
|
||||
self.check_button.setEnabled(False)
|
||||
self.select_all_button.setEnabled(False)
|
||||
self.deselect_all_button.setEnabled(False)
|
||||
self.load_settings_checkbox.setEnabled(False)
|
||||
self.edit_settings_checkbox.setEnabled(False)
|
||||
|
||||
def _toggle_all_checkboxes(self):
|
||||
"""Handles Select All and Deselect All button clicks."""
|
||||
sender = self.sender()
|
||||
check_state = Qt.Checked if sender == self.select_all_button else Qt.Unchecked
|
||||
|
||||
# Block signals to prevent triggering _handle_item_changed repeatedly
|
||||
self.list_widget.blockSignals(True)
|
||||
for i in range(self.list_widget.count()):
|
||||
item = self.list_widget.item(i)
|
||||
@@ -173,13 +175,12 @@ class UpdateCheckDialog(QDialog):
|
||||
item.setCheckState(check_state)
|
||||
self.list_widget.blockSignals(False)
|
||||
|
||||
# Manually trigger the update once after batch change
|
||||
self._handle_item_changed(None)
|
||||
|
||||
def _handle_item_changed(self, item):
|
||||
"""
|
||||
Monitors how many items are checked.
|
||||
If more than 1 item is checked, disable the 'Load Settings' checkbox.
|
||||
If more than 1 item is checked, disable the 'Enable Editing' checkbox.
|
||||
"""
|
||||
checked_count = 0
|
||||
for i in range(self.list_widget.count()):
|
||||
@@ -187,15 +188,15 @@ class UpdateCheckDialog(QDialog):
|
||||
checked_count += 1
|
||||
|
||||
if checked_count > 1:
|
||||
self.load_settings_checkbox.setChecked(False)
|
||||
self.load_settings_checkbox.setEnabled(False)
|
||||
self.load_settings_checkbox.setToolTip(
|
||||
self.edit_settings_checkbox.setChecked(False)
|
||||
self.edit_settings_checkbox.setEnabled(False)
|
||||
self.edit_settings_checkbox.setToolTip(
|
||||
self._tr("update_check_multi_selection_warning",
|
||||
"Editing settings is disabled when multiple profiles are selected.")
|
||||
)
|
||||
else:
|
||||
self.load_settings_checkbox.setEnabled(True)
|
||||
self.load_settings_checkbox.setToolTip(self._default_checkbox_tooltip)
|
||||
self.edit_settings_checkbox.setEnabled(True)
|
||||
self.edit_settings_checkbox.setToolTip(self._default_checkbox_tooltip)
|
||||
|
||||
def on_check_selected(self):
|
||||
"""Handles the 'Check Selected' button click."""
|
||||
@@ -221,6 +222,18 @@ class UpdateCheckDialog(QDialog):
|
||||
return self.selected_profiles_list
|
||||
|
||||
def should_load_into_ui(self):
|
||||
"""Returns True if the 'Load settings into UI' checkbox is checked."""
|
||||
# Only return True if it's enabled and checked (double safety)
|
||||
return self.load_settings_checkbox.isEnabled() and self.load_settings_checkbox.isChecked()
|
||||
"""
|
||||
Returns True if the settings SHOULD be loaded into the UI.
|
||||
|
||||
NEW LOGIC: Returns True if exactly ONE profile is selected.
|
||||
It does NOT care about the checkbox state anymore, because we want
|
||||
to load settings even if the user can't edit them.
|
||||
"""
|
||||
return len(self.selected_profiles_list) == 1
|
||||
|
||||
def should_enable_editing(self):
|
||||
"""
|
||||
NEW METHOD: Returns True if the user is allowed to edit the settings.
|
||||
This is linked to the checkbox.
|
||||
"""
|
||||
return self.edit_settings_checkbox.isEnabled() and self.edit_settings_checkbox.isChecked()
|
||||
@@ -1,6 +1,7 @@
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
import glob
|
||||
import queue
|
||||
import random
|
||||
import traceback
|
||||
@@ -105,6 +106,7 @@ from .classes.external_link_downloader_thread import ExternalLinkDownloadThread
|
||||
from .classes.nhentai_downloader_thread import NhentaiDownloadThread
|
||||
from .classes.downloader_factory import create_downloader_thread
|
||||
from .classes.kemono_discord_downloader_thread import KemonoDiscordDownloadThread
|
||||
from .classes.hentaifox_downloader_thread import HentaiFoxDownloadThread
|
||||
|
||||
_ff_ver = (datetime.date.today().toordinal() - 735506) // 28
|
||||
USERAGENT_FIREFOX = (f"Mozilla/5.0 (Windows NT 10.0; Win64; x64; "
|
||||
@@ -164,7 +166,7 @@ class DownloaderApp (QWidget ):
|
||||
self.is_finishing = False
|
||||
self.finish_lock = threading.Lock()
|
||||
self.add_info_in_pdf_setting = False
|
||||
|
||||
|
||||
saved_res = self.settings.value(RESOLUTION_KEY, "Auto")
|
||||
if saved_res != "Auto":
|
||||
try:
|
||||
@@ -187,6 +189,11 @@ class DownloaderApp (QWidget ):
|
||||
|
||||
self.user_data_path = user_data_path
|
||||
|
||||
self.jobs_dir = os.path.join(self.user_data_path, "jobs")
|
||||
os.makedirs(self.jobs_dir, exist_ok=True)
|
||||
self.is_running_job_queue = False
|
||||
self.current_job_file = None
|
||||
|
||||
self.config_file = os.path.join(user_data_path, "Known.txt")
|
||||
self.session_file_path = os.path.join(user_data_path, "session.json")
|
||||
self.persistent_history_file = os.path.join(user_data_path, "download_history.json")
|
||||
@@ -303,6 +310,9 @@ class DownloaderApp (QWidget ):
|
||||
self.downloaded_hash_counts_lock = threading.Lock()
|
||||
self.session_temp_files = []
|
||||
self.single_pdf_mode = False
|
||||
|
||||
self.temp_pdf_content_list = []
|
||||
self.last_effective_download_dir = None
|
||||
self.save_creator_json_enabled_this_session = True
|
||||
self.date_prefix_format = self.settings.value(DATE_PREFIX_FORMAT_KEY, "YYYY-MM-DD {post}", type=str)
|
||||
self.is_single_post_session = False
|
||||
@@ -340,7 +350,7 @@ class DownloaderApp (QWidget ):
|
||||
self.download_location_label_widget = None
|
||||
self.remove_from_filename_label_widget = None
|
||||
self.skip_words_label_widget = None
|
||||
self.setWindowTitle("Kemono Downloader v7.8.0")
|
||||
self.setWindowTitle("Kemono Downloader v7.9.1")
|
||||
setup_ui(self)
|
||||
self._connect_signals()
|
||||
if hasattr(self, 'character_input'):
|
||||
@@ -357,6 +367,176 @@ class DownloaderApp (QWidget ):
|
||||
self._check_for_interrupted_session()
|
||||
self._cleanup_after_update()
|
||||
|
||||
def add_current_settings_to_queue(self):
|
||||
"""Saves the current UI settings as a JSON job file with creator-specific paths."""
|
||||
|
||||
def get_creator_specific_path(base_dir, folder_name):
|
||||
if not folder_name:
|
||||
return base_dir
|
||||
safe_name = clean_folder_name(folder_name)
|
||||
|
||||
if base_dir.replace('\\', '/').rstrip('/').endswith(safe_name):
|
||||
return base_dir
|
||||
return os.path.join(base_dir, safe_name)
|
||||
if self.favorite_download_queue:
|
||||
count = 0
|
||||
base_settings = self._get_current_ui_settings_as_dict()
|
||||
items_to_process = list(self.favorite_download_queue)
|
||||
|
||||
for item in items_to_process:
|
||||
real_url = item.get('url')
|
||||
name = item.get('name', 'Unknown')
|
||||
|
||||
if not real_url: continue
|
||||
|
||||
job_settings = base_settings.copy()
|
||||
job_settings['api_url'] = real_url
|
||||
|
||||
# Use the name provided by the selection popup
|
||||
job_settings['output_dir'] = get_creator_specific_path(job_settings['output_dir'], name)
|
||||
|
||||
if self._save_single_job_file(job_settings, name_hint=name):
|
||||
count += 1
|
||||
|
||||
if count > 0:
|
||||
self.log_signal.emit(f"✅ Added {count} jobs to queue from selection.")
|
||||
self.link_input.clear()
|
||||
self.favorite_download_queue.clear()
|
||||
QMessageBox.information(self, "Queue", f"{count} jobs successfully added to queue!")
|
||||
else:
|
||||
QMessageBox.warning(self, "Queue Error", "Failed to add selected items to queue.")
|
||||
return
|
||||
|
||||
|
||||
url = self.link_input.text().strip()
|
||||
if not url:
|
||||
QMessageBox.warning(self, "Input Error", "Cannot add to queue: URL is empty.")
|
||||
return
|
||||
|
||||
settings = self._get_current_ui_settings_as_dict()
|
||||
settings['api_url'] = url
|
||||
|
||||
|
||||
service, user_id, post_id = extract_post_info(url)
|
||||
name_hint = "Job"
|
||||
|
||||
if service and user_id:
|
||||
|
||||
cache_key = (service.lower(), str(user_id))
|
||||
cached_name = self.creator_name_cache.get(cache_key)
|
||||
|
||||
if cached_name:
|
||||
|
||||
name_hint = cached_name
|
||||
settings['output_dir'] = get_creator_specific_path(settings['output_dir'], cached_name)
|
||||
else:
|
||||
if post_id:
|
||||
folder_name = str(post_id)
|
||||
else:
|
||||
folder_name = str(user_id)
|
||||
|
||||
name_hint = folder_name
|
||||
settings['output_dir'] = get_creator_specific_path(settings['output_dir'], folder_name)
|
||||
|
||||
if self._save_single_job_file(settings, name_hint=name_hint):
|
||||
self.log_signal.emit(f"✅ Job added to queue: {url}")
|
||||
self.link_input.clear()
|
||||
QMessageBox.information(self, "Queue", "Job successfully added to queue!")
|
||||
|
||||
def _save_single_job_file(self, settings_dict, name_hint="job"):
|
||||
"""Helper to write a single JSON job file to the jobs directory."""
|
||||
import uuid
|
||||
timestamp = int(time.time())
|
||||
unique_id = uuid.uuid4().hex[:6]
|
||||
|
||||
# Clean the name hint to be safe for filenames
|
||||
safe_name = "".join(c for c in name_hint if c.isalnum() or c in (' ', '_', '-')).strip()
|
||||
if not safe_name:
|
||||
safe_name = "job"
|
||||
|
||||
filename = f"job_{timestamp}_{safe_name}_{unique_id}.json"
|
||||
filepath = os.path.join(self.jobs_dir, filename)
|
||||
|
||||
try:
|
||||
with open(filepath, 'w', encoding='utf-8') as f:
|
||||
json.dump(settings_dict, f, indent=2)
|
||||
return True
|
||||
except Exception as e:
|
||||
self.log_signal.emit(f"❌ Failed to save job file '{filename}': {e}")
|
||||
return False
|
||||
|
||||
def execute_job_queue(self):
|
||||
"""Starts the queue processing loop."""
|
||||
job_files = sorted(glob.glob(os.path.join(self.jobs_dir, "job_*.json")))
|
||||
|
||||
if not job_files:
|
||||
QMessageBox.information(self, "Queue Empty", "No job files found in appdata/jobs.")
|
||||
return
|
||||
|
||||
|
||||
self.permanently_failed_files_for_dialog.clear()
|
||||
self._update_error_button_count()
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
self.log_signal.emit("=" * 40)
|
||||
self.log_signal.emit(f"🚀 Starting execution of {len(job_files)} queued jobs.")
|
||||
self.is_running_job_queue = True
|
||||
self.download_btn.setEnabled(False) # Disable button while running
|
||||
self.add_queue_btn.setEnabled(False)
|
||||
|
||||
self._process_next_queued_job()
|
||||
|
||||
def _process_next_queued_job(self):
|
||||
"""Loads the next job file and starts the download."""
|
||||
if self.cancellation_event.is_set():
|
||||
self.is_running_job_queue = False
|
||||
self.log_signal.emit("🛑 Queue execution cancelled.")
|
||||
self._update_button_states_and_connections()
|
||||
return
|
||||
|
||||
job_files = sorted(glob.glob(os.path.join(self.jobs_dir, "job_*.json")))
|
||||
|
||||
if not job_files:
|
||||
self.is_running_job_queue = False
|
||||
self.current_job_file = None
|
||||
self.log_signal.emit("🏁 All queued jobs finished!")
|
||||
self.link_input.clear()
|
||||
QMessageBox.information(self, "Queue Finished", "All queued jobs have been processed.")
|
||||
self._update_button_states_and_connections()
|
||||
return
|
||||
|
||||
next_job_path = job_files[0]
|
||||
self.current_job_file = next_job_path
|
||||
|
||||
self.log_signal.emit(f"📂 Loading job: {os.path.basename(next_job_path)}")
|
||||
|
||||
try:
|
||||
with open(next_job_path, 'r', encoding='utf-8') as f:
|
||||
settings = json.load(f)
|
||||
|
||||
# --- Ensure Directory Exists ---
|
||||
# The settings now contain the full path (e.g. E:/Kemono/ArtistName)
|
||||
target_dir = settings.get('output_dir', '')
|
||||
if target_dir:
|
||||
try:
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
except Exception as e:
|
||||
self.log_signal.emit(f"⚠️ Warning: Could not pre-create directory '{target_dir}': {e}")
|
||||
# -------------------------------
|
||||
|
||||
# Load settings into UI
|
||||
self._load_ui_from_settings_dict(settings)
|
||||
QCoreApplication.processEvents()
|
||||
|
||||
# Start download
|
||||
self.start_download()
|
||||
|
||||
except Exception as e:
|
||||
self.log_signal.emit(f"❌ Error loading/starting job '{next_job_path}': {e}")
|
||||
failed_path = next_job_path + ".failed"
|
||||
os.rename(next_job_path, failed_path)
|
||||
self._process_next_queued_job()
|
||||
|
||||
def _run_discord_file_download_thread(self, session, server_id, channel_id, token, output_dir, message_limit=None):
|
||||
"""
|
||||
Runs in a background thread to fetch and download all files from a Discord channel.
|
||||
@@ -660,7 +840,32 @@ class DownloaderApp (QWidget ):
|
||||
settings['add_info_in_pdf'] = self.add_info_in_pdf_setting # Save to settings dict
|
||||
settings['keep_duplicates_mode'] = self.keep_duplicates_mode
|
||||
settings['keep_duplicates_limit'] = self.keep_duplicates_limit
|
||||
|
||||
|
||||
settings['proxy_enabled'] = self.settings.value(PROXY_ENABLED_KEY, False, type=bool)
|
||||
settings['proxy_host'] = self.settings.value(PROXY_HOST_KEY, "", type=str)
|
||||
settings['proxy_port'] = self.settings.value(PROXY_PORT_KEY, "", type=str)
|
||||
settings['proxy_username'] = self.settings.value(PROXY_USERNAME_KEY, "", type=str)
|
||||
settings['proxy_password'] = self.settings.value(PROXY_PASSWORD_KEY, "", type=str)
|
||||
proxy_type_str = self.settings.value("proxy_type", "HTTP", type=str)
|
||||
|
||||
settings['proxies'] = None
|
||||
if settings['proxy_enabled'] and settings['proxy_host'] and settings['proxy_port']:
|
||||
|
||||
# Determine correct scheme
|
||||
scheme = "http"
|
||||
if proxy_type_str == "SOCKS5":
|
||||
scheme = "socks5h" # 'socks5h' forces remote DNS resolution (safer/better for bypassing)
|
||||
elif proxy_type_str == "SOCKS4":
|
||||
scheme = "socks4"
|
||||
|
||||
# Build URL string
|
||||
if settings['proxy_username'] and settings['proxy_password']:
|
||||
proxy_str = f"{scheme}://{settings['proxy_username']}:{settings['proxy_password']}@{settings['proxy_host']}:{settings['proxy_port']}"
|
||||
else:
|
||||
proxy_str = f"{scheme}://{settings['proxy_host']}:{settings['proxy_port']}"
|
||||
|
||||
settings['proxies'] = {'http': proxy_str, 'https': proxy_str}
|
||||
|
||||
return settings
|
||||
|
||||
|
||||
@@ -769,6 +974,23 @@ class DownloaderApp (QWidget ):
|
||||
|
||||
is_download_active = self._is_download_active()
|
||||
fetch_first_enabled = self.settings.value(FETCH_FIRST_KEY, False, type=bool)
|
||||
url_text = self.link_input.text().strip()
|
||||
|
||||
# --- NEW: Check for Queue Command ---
|
||||
is_queue_command = (url_text.lower() == "start queue")
|
||||
|
||||
# --- NEW: Handle 'Add to Queue' Button State ---
|
||||
if hasattr(self, 'add_queue_btn'):
|
||||
# Only enable if not downloading, URL is valid, not in queue mode, and not in specialized fetch states
|
||||
should_enable_queue = (
|
||||
not is_download_active and
|
||||
url_text != "" and
|
||||
not is_queue_command and
|
||||
not self.is_ready_to_download_fetched and
|
||||
not self.is_ready_to_download_batch_update
|
||||
)
|
||||
self.add_queue_btn.setEnabled(should_enable_queue)
|
||||
|
||||
print(f"--- DEBUG: Updating buttons (is_download_active={is_download_active}) ---")
|
||||
|
||||
if self.is_ready_to_download_fetched:
|
||||
@@ -852,7 +1074,12 @@ class DownloaderApp (QWidget ):
|
||||
self.download_btn.setText(f"⬇️ Start Download ({num_posts} Posts)")
|
||||
self.download_btn.setEnabled(True) # Keep it enabled for the user to click
|
||||
else:
|
||||
self.download_btn.setText(self._tr("start_download_button_text", "⬇️ Start Download"))
|
||||
# Check if running queue to show specific text
|
||||
if hasattr(self, 'is_running_job_queue') and self.is_running_job_queue:
|
||||
self.download_btn.setText("🔄 Processing Queue...")
|
||||
else:
|
||||
self.download_btn.setText(self._tr("start_download_button_text", "⬇️ Start Download"))
|
||||
|
||||
self.download_btn.setEnabled(False)
|
||||
|
||||
self.pause_btn.setText(self._tr("resume_download_button_text", "▶️ Resume Download") if self.is_paused else self._tr("pause_download_button_text", "⏸️ Pause Download"))
|
||||
@@ -865,22 +1092,32 @@ class DownloaderApp (QWidget ):
|
||||
self.cancel_btn.clicked.connect(self.cancel_download_button_action)
|
||||
|
||||
else:
|
||||
url_text = self.link_input.text().strip()
|
||||
_, _, post_id = extract_post_info(url_text)
|
||||
is_single_post = bool(post_id)
|
||||
|
||||
if fetch_first_enabled and not is_single_post:
|
||||
self.download_btn.setText("📄 Fetch Pages")
|
||||
# --- IDLE STATE ---
|
||||
if is_queue_command:
|
||||
# --- NEW: Queue Execution Mode ---
|
||||
self.download_btn.setText("🚀 Execute Queue")
|
||||
self.download_btn.setEnabled(True)
|
||||
# Ensure the method exists before connecting
|
||||
if hasattr(self, 'execute_job_queue'):
|
||||
self.download_btn.clicked.connect(self.execute_job_queue)
|
||||
else:
|
||||
self.download_btn.setText(self._tr("start_download_button_text", "⬇️ Start Download"))
|
||||
|
||||
self.download_btn.setEnabled(True)
|
||||
self.download_btn.clicked.connect(self.start_download)
|
||||
_, _, post_id = extract_post_info(url_text)
|
||||
is_single_post = bool(post_id)
|
||||
|
||||
if fetch_first_enabled and not is_single_post and url_text:
|
||||
self.download_btn.setText("📄 Fetch Pages")
|
||||
else:
|
||||
self.download_btn.setText(self._tr("start_download_button_text", "⬇️ Start Download"))
|
||||
|
||||
self.download_btn.setEnabled(True)
|
||||
self.download_btn.clicked.connect(self.start_download)
|
||||
|
||||
self.pause_btn.setText(self._tr("pause_download_button_text", "⏸️ Pause Download"))
|
||||
self.pause_btn.setEnabled(False)
|
||||
self.cancel_btn.setText(self._tr("cancel_button_text", "❌ Cancel & Reset UI"))
|
||||
self.cancel_btn.setEnabled(False)
|
||||
|
||||
|
||||
def _run_fetch_only_thread(self, fetch_args):
|
||||
"""
|
||||
Runs in a background thread to ONLY fetch all posts without downloading.
|
||||
@@ -2735,6 +2972,25 @@ class DownloaderApp (QWidget ):
|
||||
else:
|
||||
self.log_signal.emit("ℹ️ Link export was cancelled by the user.")
|
||||
|
||||
def _set_inputs_read_only(self, read_only):
|
||||
"""Disables input fields (Read-Only mode) but keeps action buttons enabled."""
|
||||
# List of widgets to disable in Read-Only mode
|
||||
widgets_to_lock = [
|
||||
self.link_input, self.dir_input, self.character_input,
|
||||
self.skip_words_input, self.remove_from_filename_input,
|
||||
self.custom_folder_input, self.cookie_text_input,
|
||||
self.thread_count_input, self.start_page_input, self.end_page_input,
|
||||
self.use_subfolders_checkbox, self.use_subfolder_per_post_checkbox,
|
||||
self.skip_zip_checkbox, self.download_thumbnails_checkbox,
|
||||
self.compress_images_checkbox, self.scan_content_images_checkbox,
|
||||
self.use_cookie_checkbox, self.manga_mode_checkbox,
|
||||
self.radio_all, self.radio_images, self.radio_videos,
|
||||
self.char_filter_scope_toggle_button, self.skip_scope_toggle_button
|
||||
]
|
||||
|
||||
for widget in widgets_to_lock:
|
||||
if widget:
|
||||
widget.setEnabled(not read_only)
|
||||
|
||||
def get_filter_mode (self ):
|
||||
if self.radio_more and self.radio_more.isChecked():
|
||||
@@ -3003,7 +3259,6 @@ class DownloaderApp (QWidget ):
|
||||
if self.single_pdf_setting:
|
||||
self.use_subfolder_per_post_checkbox.setChecked(False)
|
||||
|
||||
# --- Logging ---
|
||||
self.log_signal.emit(f"ℹ️ 'More' filter set: {scope_text}, Format: {self.text_export_format.upper()}")
|
||||
if is_any_pdf_mode:
|
||||
status_single = "Enabled" if self.single_pdf_setting else "Disabled"
|
||||
@@ -3012,19 +3267,18 @@ class DownloaderApp (QWidget ):
|
||||
self.log_signal.emit(" ↳ Multithreading disabled for PDF export.")
|
||||
|
||||
else:
|
||||
# --- User clicked Cancel: Revert to default ---
|
||||
self.log_signal.emit("ℹ️ 'More' filter selection cancelled. Reverting to 'All'.")
|
||||
if hasattr(self, 'radio_all'):
|
||||
self.radio_all.setChecked(True)
|
||||
|
||||
# Case 2: Switched AWAY from the "More" button (e.g., clicked 'Images' or 'All')
|
||||
|
||||
elif button != self.radio_more and checked:
|
||||
self.radio_more.setText("More")
|
||||
self.more_filter_scope = None
|
||||
self.single_pdf_setting = False
|
||||
self.add_info_in_pdf_setting = False # Reset setting
|
||||
|
||||
# Restore enabled states for options that PDF mode might have disabled
|
||||
|
||||
if hasattr(self, 'use_multithreading_checkbox'):
|
||||
self.use_multithreading_checkbox.setEnabled(True)
|
||||
self._update_multithreading_for_date_mode() # Re-check manga logic
|
||||
@@ -3668,7 +3922,11 @@ class DownloaderApp (QWidget ):
|
||||
'txt_file': 'coomer.txt',
|
||||
'url_regex': r'https?://(?:www\.)?coomer\.(?:su|party|st)/[^/\s]+/user/[^/\s]+(?:/post/\d+)?/?'
|
||||
},
|
||||
|
||||
'hentaifox.com': {
|
||||
'name': 'HentaiFox',
|
||||
'txt_file': 'hentaifox.txt',
|
||||
'url_regex': r'https?://(?:www\.)?hentaifox\.com/(?:g|gallery)/\d+/?'
|
||||
},
|
||||
'allporncomic.com': {
|
||||
'name': 'AllPornComic',
|
||||
'txt_file': 'allporncomic.txt',
|
||||
@@ -3749,7 +4007,8 @@ class DownloaderApp (QWidget ):
|
||||
'toonily.com', 'toonily.me',
|
||||
'hentai2read.com',
|
||||
'saint2.su', 'saint2.pk',
|
||||
'imgur.com', 'bunkr.'
|
||||
'imgur.com', 'bunkr.',
|
||||
'hentaifox.com'
|
||||
]
|
||||
|
||||
for url in urls_to_download:
|
||||
@@ -3826,6 +4085,10 @@ class DownloaderApp (QWidget ):
|
||||
self.downloaded_hash_counts.clear()
|
||||
|
||||
if not is_restore and not is_continuation:
|
||||
|
||||
if not self.is_running_job_queue:
|
||||
self.permanently_failed_files_for_dialog.clear()
|
||||
|
||||
self.permanently_failed_files_for_dialog.clear()
|
||||
|
||||
self.retryable_failed_files_info.clear()
|
||||
@@ -3833,6 +4096,7 @@ class DownloaderApp (QWidget ):
|
||||
|
||||
self._clear_stale_temp_files()
|
||||
self.session_temp_files = []
|
||||
self.temp_pdf_content_list = []
|
||||
|
||||
processed_post_ids_for_restore = []
|
||||
manga_counters_for_restore = None
|
||||
@@ -3916,6 +4180,7 @@ class DownloaderApp (QWidget ):
|
||||
return False
|
||||
effective_output_dir_for_run = os.path.normpath(main_ui_download_dir) if main_ui_download_dir else ""
|
||||
|
||||
self.last_effective_download_dir = effective_output_dir_for_run
|
||||
if not is_restore:
|
||||
self._create_initial_session_file(api_url, effective_output_dir_for_run, remaining_queue=self.favorite_download_queue)
|
||||
|
||||
@@ -3939,9 +4204,12 @@ class DownloaderApp (QWidget ):
|
||||
|
||||
self.cancellation_message_logged_this_session = False
|
||||
|
||||
# START of the new refactored block
|
||||
service, id1, id2 = extract_post_info(api_url)
|
||||
|
||||
# [NEW] Get proxy settings immediately
|
||||
ui_settings = self._get_current_ui_settings_as_dict()
|
||||
proxies_to_use = ui_settings.get('proxies')
|
||||
|
||||
specialized_thread = create_downloader_thread(
|
||||
main_app=self,
|
||||
api_url=api_url,
|
||||
@@ -3964,18 +4232,18 @@ class DownloaderApp (QWidget ):
|
||||
|
||||
self.set_ui_enabled(False)
|
||||
self.download_thread = specialized_thread
|
||||
|
||||
# [NEW] Inject proxies into the thread manually
|
||||
if hasattr(self.download_thread, 'proxies'):
|
||||
self.download_thread.proxies = proxies_to_use
|
||||
|
||||
self._connect_specialized_thread_signals(self.download_thread)
|
||||
self.download_thread.start()
|
||||
self._update_button_states_and_connections()
|
||||
return True
|
||||
# END of the new refactored block
|
||||
|
||||
if not service or not id1:
|
||||
QMessageBox.critical(self, "Input Error", "Invalid or unsupported URL format.")
|
||||
return False
|
||||
|
||||
user_id, post_id_from_url = id1, id2
|
||||
|
||||
|
||||
if direct_api_url and not post_id_from_url and item_type_from_queue and 'post' in item_type_from_queue:
|
||||
self.log_signal.emit(f"❌ CRITICAL ERROR: Could not parse post ID from the queued POST URL: {api_url}")
|
||||
self.log_signal.emit(" Skipping this item. This might be due to an unsupported URL format or a temporary issue.")
|
||||
@@ -4408,6 +4676,14 @@ class DownloaderApp (QWidget ):
|
||||
if should_use_multithreading_for_posts:
|
||||
log_messages.append(f" Number of Post Worker Threads: {effective_num_post_workers}")
|
||||
|
||||
proxy_enabled_log = self.settings.value(PROXY_ENABLED_KEY, False, type=bool)
|
||||
if proxy_enabled_log:
|
||||
p_host = self.settings.value(PROXY_HOST_KEY, "")
|
||||
p_port = self.settings.value(PROXY_PORT_KEY, "")
|
||||
log_messages.append(f" Proxy: Enabled ({p_host}:{p_port})")
|
||||
else:
|
||||
log_messages.append(f" Proxy: Disabled")
|
||||
|
||||
if domain_override_command:
|
||||
self.log_signal.emit(f"ℹ️ Domain Override Active: Will probe for the correct 'n*' subdomain on the '.{domain_override_command}' domain for each file.")
|
||||
|
||||
@@ -4420,7 +4696,7 @@ class DownloaderApp (QWidget ):
|
||||
self.set_ui_enabled(False)
|
||||
|
||||
from src.config.constants import FOLDER_NAME_STOP_WORDS
|
||||
|
||||
current_proxies = self._get_current_ui_settings_as_dict().get('proxies')
|
||||
args_template = {
|
||||
'api_url_input': api_url,
|
||||
'download_root': effective_output_dir_for_run,
|
||||
@@ -4496,7 +4772,8 @@ class DownloaderApp (QWidget ):
|
||||
'fetch_first': fetch_first_enabled,
|
||||
'sfp_threshold': download_commands.get('sfp_threshold'),
|
||||
'handle_unknown_mode': handle_unknown_command,
|
||||
'add_info_in_pdf': self.add_info_in_pdf_setting,
|
||||
'add_info_in_pdf': self.add_info_in_pdf_setting,
|
||||
'proxies': current_proxies
|
||||
}
|
||||
|
||||
args_template['override_output_dir'] = override_output_dir
|
||||
@@ -4522,7 +4799,8 @@ class DownloaderApp (QWidget ):
|
||||
'app_base_dir': app_base_dir_for_cookies,
|
||||
'manga_filename_style_for_sort_check': self.manga_filename_style,
|
||||
'processed_post_ids': processed_post_ids_for_this_run,
|
||||
'fetch_all_first': True
|
||||
'fetch_all_first': True,
|
||||
'proxies': self._get_current_ui_settings_as_dict().get('proxies')
|
||||
}
|
||||
|
||||
self.download_thread = threading.Thread(target=self._run_fetch_only_thread, args=(fetch_thread_args,), daemon=True)
|
||||
@@ -4847,8 +5125,54 @@ class DownloaderApp (QWidget ):
|
||||
self.is_ready_to_download_batch_update = True
|
||||
|
||||
self.progress_label.setText(f"Found {total_posts} new posts. Ready to download.")
|
||||
self.set_ui_enabled(True) # Re-enable UI
|
||||
self._update_button_states_and_connections() # Update buttons to "Start Download (X)"
|
||||
self.set_ui_enabled(True) # Re-enable UI first
|
||||
|
||||
# [NEW] Apply Read-Only mode if it was selected in the dialog
|
||||
if getattr(self, 'update_settings_read_only_mode', False):
|
||||
self._set_inputs_read_only(True)
|
||||
|
||||
self._update_button_states_and_connections()
|
||||
|
||||
def _show_update_check_dialog(self):
|
||||
"""Shows the Update Check Dialog and applies Load/Edit logic."""
|
||||
if self.is_restore_pending:
|
||||
QMessageBox.warning(self, "Restore Pending", "Please restore or discard the previous session first.")
|
||||
return
|
||||
|
||||
dialog = UpdateCheckDialog(self.user_data_path, self, self)
|
||||
|
||||
if dialog.exec_() == QDialog.Accepted:
|
||||
profiles = dialog.get_selected_profiles()
|
||||
if not profiles: return
|
||||
|
||||
self.active_update_profiles_list = profiles
|
||||
|
||||
# --- LOGIC START ---
|
||||
|
||||
# 1. ALWAYS Load Settings if appropriate (e.g. Single Profile selected)
|
||||
# The dialog now returns True for should_load_into_ui() if count == 1, regardless of checkbox
|
||||
if dialog.should_load_into_ui():
|
||||
# Load settings from the FIRST selected profile
|
||||
first_profile_settings = profiles[0]['data'].get('settings', {})
|
||||
self._load_ui_from_settings_dict(first_profile_settings)
|
||||
|
||||
# 2. Check if Editing is Allowed
|
||||
if dialog.should_enable_editing():
|
||||
self.update_settings_read_only_mode = False
|
||||
self.override_update_profile_settings = True # Use UI values for download
|
||||
self.log_signal.emit("ℹ️ Settings loaded in EDITABLE mode.")
|
||||
else:
|
||||
self.update_settings_read_only_mode = True
|
||||
self.override_update_profile_settings = False # Use original JSON values (safer for Read-Only)
|
||||
self.log_signal.emit("ℹ️ Settings loaded in READ-ONLY mode.")
|
||||
else:
|
||||
# Multiple profiles or load disabled
|
||||
self.update_settings_read_only_mode = False
|
||||
self.override_update_profile_settings = False
|
||||
|
||||
# --- LOGIC END ---
|
||||
|
||||
self._start_batch_update_check(self.active_update_profiles_list)
|
||||
|
||||
def _start_download_of_batch_update(self):
|
||||
"""
|
||||
@@ -4878,8 +5202,7 @@ class DownloaderApp (QWidget ):
|
||||
|
||||
ppw_expected_keys = list(PostProcessorWorker.__init__.__code__.co_varnames)[1:]
|
||||
|
||||
# 1. Define all LIVE RUNTIME arguments.
|
||||
# These are taken from the current app state and are the same for all workers.
|
||||
current_proxies = self._get_current_ui_settings_as_dict().get('proxies')
|
||||
live_runtime_args = {
|
||||
'emitter': self.worker_to_gui_queue,
|
||||
'creator_name_cache': self.creator_name_cache,
|
||||
@@ -4909,7 +5232,8 @@ class DownloaderApp (QWidget ):
|
||||
'use_cookie': self.use_cookie_checkbox.isChecked(),
|
||||
'cookie_text': self.cookie_text_input.text(),
|
||||
'selected_cookie_file': self.selected_cookie_filepath,
|
||||
'add_info_in_pdf': self.add_info_in_pdf_setting,
|
||||
'add_info_in_pdf': self.add_info_in_pdf_setting,
|
||||
'proxies': current_proxies,
|
||||
}
|
||||
|
||||
# 2. Define DEFAULTS for all settings that *should* be in the profile.
|
||||
@@ -5145,6 +5469,19 @@ class DownloaderApp (QWidget ):
|
||||
self._update_manga_filename_style_button_text()
|
||||
self._update_multipart_toggle_button_text()
|
||||
|
||||
if 'proxy_enabled' in settings:
|
||||
self.settings.setValue(PROXY_ENABLED_KEY, settings['proxy_enabled'])
|
||||
if 'proxy_host' in settings:
|
||||
self.settings.setValue(PROXY_HOST_KEY, settings['proxy_host'])
|
||||
if 'proxy_port' in settings:
|
||||
self.settings.setValue(PROXY_PORT_KEY, settings['proxy_port'])
|
||||
if 'proxy_username' in settings:
|
||||
self.settings.setValue(PROXY_USERNAME_KEY, settings['proxy_username'])
|
||||
if 'proxy_password' in settings:
|
||||
self.settings.setValue(PROXY_PASSWORD_KEY, settings['proxy_password'])
|
||||
|
||||
self.settings.sync()
|
||||
|
||||
def start_multi_threaded_download(self, num_post_workers, **kwargs):
|
||||
"""
|
||||
Initializes and starts the multi-threaded download process.
|
||||
@@ -5187,8 +5524,13 @@ class DownloaderApp (QWidget ):
|
||||
global PostProcessorWorker, download_from_api
|
||||
|
||||
worker_args_template = fetcher_args['worker_args_template']
|
||||
logger_func = lambda msg: self.log_signal.emit(f"[Fetcher] {msg}")
|
||||
|
||||
def logger_func(msg):
|
||||
try:
|
||||
import sip
|
||||
if not sip.isdeleted(self):
|
||||
self.log_signal.emit(f"[Fetcher] {msg}")
|
||||
except (RuntimeError, ImportError, AttributeError):
|
||||
pass # Window is gone, ignore logging
|
||||
try:
|
||||
# This single call now handles all fetching logic, including 'Fetch First'.
|
||||
post_generator = download_from_api(
|
||||
@@ -5205,7 +5547,8 @@ class DownloaderApp (QWidget ):
|
||||
app_base_dir=worker_args_template.get('app_base_dir'),
|
||||
manga_filename_style_for_sort_check=worker_args_template.get('manga_filename_style'),
|
||||
processed_post_ids=worker_args_template.get('processed_post_ids', []),
|
||||
fetch_all_first=worker_args_template.get('fetch_first', False)
|
||||
fetch_all_first=worker_args_template.get('fetch_first', False),
|
||||
proxies=worker_args_template.get('proxies')
|
||||
)
|
||||
|
||||
ppw_expected_keys = list(PostProcessorWorker.__init__.__code__.co_varnames)[1:]
|
||||
@@ -5268,8 +5611,18 @@ class DownloaderApp (QWidget ):
|
||||
permanent, history_data,
|
||||
temp_filepath) = result_tuple
|
||||
|
||||
if temp_filepath: self.session_temp_files.append(temp_filepath)
|
||||
|
||||
if temp_filepath:
|
||||
self.session_temp_files.append(temp_filepath)
|
||||
|
||||
# If Single PDF mode is enabled, we need to load the data
|
||||
# from the temp file into memory for the final aggregation.
|
||||
if self.single_pdf_setting:
|
||||
try:
|
||||
with open(temp_filepath, 'r', encoding='utf-8') as f:
|
||||
post_content_data = json.load(f)
|
||||
self.temp_pdf_content_list.append(post_content_data)
|
||||
except Exception as e:
|
||||
self.log_signal.emit(f"⚠️ Error reading temp file for PDF aggregation: {e}")
|
||||
with self.downloaded_files_lock:
|
||||
self.download_counter += downloaded
|
||||
self.skip_counter += skipped
|
||||
@@ -5295,47 +5648,73 @@ class DownloaderApp (QWidget ):
|
||||
self.finished_signal.emit(self.download_counter, self.skip_counter, self.cancellation_event.is_set(), self.all_kept_original_filenames)
|
||||
|
||||
def _trigger_single_pdf_creation(self):
|
||||
"""Reads temp files, sorts them by date, then creates the single PDF."""
|
||||
self.log_signal.emit("="*40)
|
||||
self.log_signal.emit("Creating single PDF from collected text files...")
|
||||
|
||||
posts_content_data = []
|
||||
for temp_filepath in self.session_temp_files:
|
||||
try:
|
||||
with open(temp_filepath, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
posts_content_data.append(data)
|
||||
except Exception as e:
|
||||
self.log_signal.emit(f" ⚠️ Could not read temp file '{temp_filepath}': {e}")
|
||||
|
||||
if not posts_content_data:
|
||||
self.log_signal.emit(" No content was collected. Aborting PDF creation.")
|
||||
"""
|
||||
Triggers the creation of a single PDF from collected text content in a BACKGROUND THREAD.
|
||||
"""
|
||||
if not self.temp_pdf_content_list:
|
||||
self.log_signal.emit("⚠️ No content collected for Single PDF.")
|
||||
return
|
||||
|
||||
output_dir = self.dir_input.text().strip() or QStandardPaths.writableLocation(QStandardPaths.DownloadLocation)
|
||||
default_filename = os.path.join(output_dir, "Consolidated_Content.pdf")
|
||||
filepath, _ = QFileDialog.getSaveFileName(self, "Save Single PDF", default_filename, "PDF Files (*.pdf)")
|
||||
# 1. Sort the content
|
||||
self.log_signal.emit(" Sorting collected content for PDF...")
|
||||
def sort_key(post):
|
||||
p_date = post.get('published') or "0000-00-00"
|
||||
a_date = post.get('added') or "0000-00-00"
|
||||
pid = post.get('id') or "0"
|
||||
return (p_date, a_date, pid)
|
||||
|
||||
if not filepath:
|
||||
self.log_signal.emit(" Single PDF creation cancelled by user.")
|
||||
return
|
||||
sorted_content = sorted(self.temp_pdf_content_list, key=sort_key)
|
||||
|
||||
if not filepath.lower().endswith('.pdf'):
|
||||
filepath += '.pdf'
|
||||
# 2. Determine Filename
|
||||
first_post = sorted_content[0]
|
||||
creator_name = first_post.get('creator_name') or first_post.get('user') or "Unknown_Creator"
|
||||
clean_creator = clean_folder_name(creator_name)
|
||||
|
||||
filename = f"[{clean_creator}] Complete_Collection.pdf"
|
||||
|
||||
# --- FIX 3: Corrected Fallback Logic ---
|
||||
# Use the stored dir, or fall back to the text input in the UI, or finally the app root
|
||||
base_dir = self.last_effective_download_dir
|
||||
if not base_dir:
|
||||
base_dir = self.dir_input.text().strip()
|
||||
if not base_dir:
|
||||
base_dir = self.app_base_dir
|
||||
|
||||
output_path = os.path.join(base_dir, filename)
|
||||
# ---------------------------------------
|
||||
|
||||
# 3. Get Options
|
||||
font_path = os.path.join(self.app_base_dir, 'data', 'dejavu-sans', 'DejaVuSans.ttf')
|
||||
|
||||
self.log_signal.emit(" Sorting collected posts by date (oldest first)...")
|
||||
sorted_content = sorted(posts_content_data, key=lambda x: x.get('published', 'Z'))
|
||||
# Get 'Add Info Page' preference
|
||||
add_info = True
|
||||
if hasattr(self, 'more_options_dialog') and self.more_options_dialog:
|
||||
add_info = self.more_options_dialog.get_add_info_state()
|
||||
elif hasattr(self, 'add_info_in_pdf_setting'):
|
||||
add_info = self.add_info_in_pdf_setting
|
||||
|
||||
create_single_pdf_from_content(
|
||||
sorted_content,
|
||||
filepath,
|
||||
font_path,
|
||||
add_info_page=self.add_info_in_pdf_setting, # Pass the flag here
|
||||
logger=self.log_signal.emit
|
||||
# 4. START THE THREAD
|
||||
self.pdf_thread = PdfGenerationThread(
|
||||
posts_data=sorted_content,
|
||||
output_filename=output_path,
|
||||
font_path=font_path,
|
||||
add_info_page=add_info,
|
||||
logger_func=self.log_signal.emit
|
||||
)
|
||||
self.log_signal.emit("="*40)
|
||||
|
||||
self.pdf_thread.finished_signal.connect(self._on_pdf_generation_finished)
|
||||
self.pdf_thread.start()
|
||||
|
||||
def _on_pdf_generation_finished(self, success, message):
|
||||
"""Callback for when the PDF thread is done."""
|
||||
if success:
|
||||
self.log_signal.emit(f"✅ {message}")
|
||||
QMessageBox.information(self, "PDF Created", message)
|
||||
else:
|
||||
self.log_signal.emit(f"❌ PDF Creation Error: {message}")
|
||||
QMessageBox.warning(self, "PDF Error", f"Could not create PDF: {message}")
|
||||
|
||||
# Optional: Clear the temp list now that we are done
|
||||
self.temp_pdf_content_list = []
|
||||
|
||||
def _add_to_history_candidates(self, history_data):
|
||||
"""Adds processed post data to the history candidates list and updates the creator profile."""
|
||||
@@ -5732,9 +6111,7 @@ class DownloaderApp (QWidget ):
|
||||
if not self.finish_lock.acquire(blocking=False):
|
||||
return
|
||||
|
||||
# --- Flag to track if we still hold the lock ---
|
||||
lock_held = True
|
||||
# ----------------------------------------------------
|
||||
|
||||
try:
|
||||
if self.is_finishing:
|
||||
@@ -5743,6 +6120,14 @@ class DownloaderApp (QWidget ):
|
||||
|
||||
if cancelled_by_user:
|
||||
self.log_signal.emit("✅ Cancellation complete. Resetting UI.")
|
||||
|
||||
# --- NEW: Reset Queue State on Cancel ---
|
||||
if getattr(self, 'is_running_job_queue', False):
|
||||
self.log_signal.emit("🛑 Queue execution stopped by user.")
|
||||
self.is_running_job_queue = False
|
||||
self.current_job_file = None
|
||||
# ----------------------------------------
|
||||
|
||||
self._clear_session_file()
|
||||
self.interrupted_session_data = None
|
||||
self.is_restore_pending = False
|
||||
@@ -5757,7 +6142,7 @@ class DownloaderApp (QWidget ):
|
||||
|
||||
self.log_signal.emit("🏁 Download of current item complete.")
|
||||
|
||||
# --- QUEUE PROCESSING BLOCK ---
|
||||
# --- EXISTING: FAVORITE QUEUE PROCESSING BLOCK ---
|
||||
if self.is_processing_favorites_queue and self.favorite_download_queue:
|
||||
self.log_signal.emit("✅ Item finished. Processing next in queue...")
|
||||
if self.download_thread and isinstance(self.download_thread, QThread):
|
||||
@@ -5765,13 +6150,46 @@ class DownloaderApp (QWidget ):
|
||||
self.download_thread = None
|
||||
self.is_finishing = False
|
||||
|
||||
# FIX: Manual release + update flag
|
||||
self.finish_lock.release()
|
||||
lock_held = False
|
||||
|
||||
self._process_next_favorite_download()
|
||||
return
|
||||
# ---------------------------------------------------------
|
||||
|
||||
if getattr(self, 'is_running_job_queue', False) and getattr(self, 'current_job_file', None):
|
||||
self.log_signal.emit(f"✅ Job finished. Deleting job file: {os.path.basename(self.current_job_file)}")
|
||||
|
||||
if self.retryable_failed_files_info:
|
||||
self.log_signal.emit(f"⚠️ Job had {len(self.retryable_failed_files_info)} incomplete files. Adding to cumulative error report.")
|
||||
self.permanently_failed_files_for_dialog.extend(self.retryable_failed_files_info)
|
||||
self._update_error_button_count()
|
||||
self.retryable_failed_files_info.clear()
|
||||
|
||||
self._finalize_download_history()
|
||||
if self.thread_pool:
|
||||
self.thread_pool.shutdown(wait=False)
|
||||
self.thread_pool = None
|
||||
self._cleanup_temp_files()
|
||||
self.single_pdf_setting = False # Reset per job
|
||||
|
||||
# 2. Delete the finished job file so it isn't run again
|
||||
try:
|
||||
if os.path.exists(self.current_job_file):
|
||||
os.remove(self.current_job_file)
|
||||
except Exception as e:
|
||||
self.log_signal.emit(f"⚠️ Failed to delete finished job file: {e}")
|
||||
|
||||
# 3. Reset state for next job
|
||||
self.current_job_file = None
|
||||
self.is_finishing = False
|
||||
|
||||
# 4. Release lock
|
||||
self.finish_lock.release()
|
||||
lock_held = False
|
||||
|
||||
# 5. Trigger next job in queue (using QTimer to allow stack to unwind)
|
||||
QTimer.singleShot(100, self._process_next_queued_job)
|
||||
return
|
||||
|
||||
if self.is_processing_favorites_queue:
|
||||
self.is_processing_favorites_queue = False
|
||||
@@ -5888,12 +6306,21 @@ class DownloaderApp (QWidget ):
|
||||
|
||||
# Reset the finishing lock and exit to let the retry session take over
|
||||
self.is_finishing = False
|
||||
|
||||
# Release lock here as we are returning
|
||||
self.finish_lock.release()
|
||||
lock_held = False
|
||||
return
|
||||
|
||||
self.is_fetcher_thread_running = False
|
||||
|
||||
# --- POST DOWNLOAD ACTION (Only if queue is finished or not running queue) ---
|
||||
if not cancelled_by_user and not self.is_processing_favorites_queue:
|
||||
self._execute_post_download_action()
|
||||
# If we were running a job queue, we only do this when the queue is EMPTY (handled by _process_next_queued_job)
|
||||
# But since we return early for job queue continuation above, getting here means
|
||||
# we are either in a standard download OR the job queue has finished/was cancelled.
|
||||
if not getattr(self, 'is_running_job_queue', False):
|
||||
self._execute_post_download_action()
|
||||
|
||||
self.set_ui_enabled(True)
|
||||
self._update_button_states_and_connections()
|
||||
@@ -7088,4 +7515,36 @@ class DownloaderApp (QWidget ):
|
||||
|
||||
if not success_starting_download:
|
||||
self.log_signal.emit(f"⚠️ Failed to initiate download for '{item_display_name}'. Skipping and moving to the next item in queue.")
|
||||
QTimer.singleShot(100, self._process_next_favorite_download)
|
||||
QTimer.singleShot(100, self._process_next_favorite_download)
|
||||
|
||||
class PdfGenerationThread(QThread):
|
||||
finished_signal = pyqtSignal(bool, str) # success, message
|
||||
|
||||
def __init__(self, posts_data, output_filename, font_path, add_info_page, logger_func):
|
||||
super().__init__()
|
||||
self.posts_data = posts_data
|
||||
self.output_filename = output_filename
|
||||
self.font_path = font_path
|
||||
self.add_info_page = add_info_page
|
||||
self.logger_func = logger_func
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
from .dialogs.SinglePDF import create_single_pdf_from_content
|
||||
self.logger_func("📄 Background Task: Generating Single PDF... (This may take a while)")
|
||||
|
||||
success = create_single_pdf_from_content(
|
||||
self.posts_data,
|
||||
self.output_filename,
|
||||
self.font_path,
|
||||
self.add_info_page,
|
||||
logger=self.logger_func
|
||||
)
|
||||
|
||||
if success:
|
||||
self.finished_signal.emit(True, f"PDF Saved: {os.path.basename(self.output_filename)}")
|
||||
else:
|
||||
self.finished_signal.emit(False, "PDF generation failed.")
|
||||
|
||||
except Exception as e:
|
||||
self.finished_signal.emit(False, str(e))
|
||||
@@ -347,7 +347,6 @@ def setup_ui(main_app):
|
||||
left_layout.addLayout(checkboxes_group_layout)
|
||||
|
||||
# --- Action Buttons & Remaining UI ---
|
||||
# ... (The rest of the setup_ui function remains unchanged)
|
||||
main_app.standard_action_buttons_widget = QWidget()
|
||||
btn_layout = QHBoxLayout(main_app.standard_action_buttons_widget)
|
||||
btn_layout.setContentsMargins(0, 10, 0, 0)
|
||||
@@ -357,6 +356,11 @@ def setup_ui(main_app):
|
||||
font.setBold(True)
|
||||
main_app.download_btn.setFont(font)
|
||||
main_app.download_btn.clicked.connect(main_app.start_download)
|
||||
|
||||
main_app.add_queue_btn = QPushButton("➕ Add to Queue")
|
||||
main_app.add_queue_btn.setToolTip("Save current settings as a job for later execution.")
|
||||
main_app.add_queue_btn.clicked.connect(main_app.add_current_settings_to_queue)
|
||||
|
||||
main_app.pause_btn = QPushButton("⏸️ Pause Download")
|
||||
main_app.pause_btn.setEnabled(False)
|
||||
main_app.pause_btn.clicked.connect(main_app._handle_pause_resume_action)
|
||||
@@ -367,6 +371,7 @@ def setup_ui(main_app):
|
||||
main_app.error_btn.setToolTip("View files skipped due to errors and optionally retry them.")
|
||||
main_app.error_btn.setEnabled(True)
|
||||
btn_layout.addWidget(main_app.download_btn)
|
||||
btn_layout.addWidget(main_app.add_queue_btn)
|
||||
btn_layout.addWidget(main_app.pause_btn)
|
||||
btn_layout.addWidget(main_app.cancel_btn)
|
||||
btn_layout.addWidget(main_app.error_btn)
|
||||
|
||||
Reference in New Issue
Block a user