mirror of
https://github.com/Yuvi9587/Kemono-Downloader.git
synced 2025-12-29 16:14:44 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
56a83195b2 |
@@ -99,7 +99,7 @@ Built with PyQt5, this tool is designed for users who want deep filtering capabi
|
|||||||
### Install Dependencies
|
### Install Dependencies
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pip install PyQt5 requests Pillow mega.py fpdf2 python-docx
|
pip install PyQt5 requests Pillow mega.py fpdf python-docx
|
||||||
```
|
```
|
||||||
|
|
||||||
### Running the Application
|
### Running the Application
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ import traceback
|
|||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
import json
|
import json
|
||||||
import requests
|
import requests
|
||||||
import cloudscraper
|
|
||||||
from ..utils.network_utils import extract_post_info, prepare_cookies_for_request
|
from ..utils.network_utils import extract_post_info, prepare_cookies_for_request
|
||||||
from ..config.constants import (
|
from ..config.constants import (
|
||||||
STYLE_DATE_POST_TITLE
|
STYLE_DATE_POST_TITLE
|
||||||
@@ -13,6 +12,7 @@ from ..config.constants import (
|
|||||||
def fetch_posts_paginated(api_url_base, headers, offset, logger, cancellation_event=None, pause_event=None, cookies_dict=None):
|
def fetch_posts_paginated(api_url_base, headers, offset, logger, cancellation_event=None, pause_event=None, cookies_dict=None):
|
||||||
"""
|
"""
|
||||||
Fetches a single page of posts from the API with robust retry logic.
|
Fetches a single page of posts from the API with robust retry logic.
|
||||||
|
NEW: Requests only essential fields to keep the response size small and reliable.
|
||||||
"""
|
"""
|
||||||
if cancellation_event and cancellation_event.is_set():
|
if cancellation_event and cancellation_event.is_set():
|
||||||
raise RuntimeError("Fetch operation cancelled by user.")
|
raise RuntimeError("Fetch operation cancelled by user.")
|
||||||
@@ -33,7 +33,7 @@ def fetch_posts_paginated(api_url_base, headers, offset, logger, cancellation_ev
|
|||||||
if cancellation_event and cancellation_event.is_set():
|
if cancellation_event and cancellation_event.is_set():
|
||||||
raise RuntimeError("Fetch operation cancelled by user during retry loop.")
|
raise RuntimeError("Fetch operation cancelled by user during retry loop.")
|
||||||
|
|
||||||
log_message = f" Fetching post list: {paginated_url} (Page approx. {offset // 50 + 1})"
|
log_message = f" Fetching post list: {api_url_base}?o={offset} (Page approx. {offset // 50 + 1})"
|
||||||
if attempt > 0:
|
if attempt > 0:
|
||||||
log_message += f" (Attempt {attempt + 1}/{max_retries})"
|
log_message += f" (Attempt {attempt + 1}/{max_retries})"
|
||||||
logger(log_message)
|
logger(log_message)
|
||||||
@@ -41,23 +41,9 @@ def fetch_posts_paginated(api_url_base, headers, offset, logger, cancellation_ev
|
|||||||
try:
|
try:
|
||||||
response = requests.get(paginated_url, headers=headers, timeout=(15, 60), cookies=cookies_dict)
|
response = requests.get(paginated_url, headers=headers, timeout=(15, 60), cookies=cookies_dict)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
response.encoding = 'utf-8'
|
|
||||||
return response.json()
|
return response.json()
|
||||||
|
|
||||||
except requests.exceptions.RequestException as e:
|
except requests.exceptions.RequestException as e:
|
||||||
# Handle 403 error on the FIRST page as a rate limit/block
|
|
||||||
if e.response is not None and e.response.status_code == 403 and offset == 0:
|
|
||||||
logger(" ❌ Access Denied (403 Forbidden) on the first page.")
|
|
||||||
logger(" This is likely a rate limit or a Cloudflare block.")
|
|
||||||
logger(" 💡 SOLUTION: Wait a while, use a VPN, or provide a valid session cookie.")
|
|
||||||
return [] # Stop the process gracefully
|
|
||||||
|
|
||||||
# Handle 400 error as the end of pages
|
|
||||||
if e.response is not None and e.response.status_code == 400:
|
|
||||||
logger(f" ✅ Reached end of posts (API returned 400 Bad Request for offset {offset}).")
|
|
||||||
return []
|
|
||||||
|
|
||||||
# Handle all other network errors with a retry
|
|
||||||
logger(f" ⚠️ Retryable network error on page fetch (Attempt {attempt + 1}): {e}")
|
logger(f" ⚠️ Retryable network error on page fetch (Attempt {attempt + 1}): {e}")
|
||||||
if attempt < max_retries - 1:
|
if attempt < max_retries - 1:
|
||||||
delay = retry_delay * (2 ** attempt)
|
delay = retry_delay * (2 ** attempt)
|
||||||
@@ -79,28 +65,26 @@ def fetch_posts_paginated(api_url_base, headers, offset, logger, cancellation_ev
|
|||||||
|
|
||||||
raise RuntimeError(f"Failed to fetch page {paginated_url} after all attempts.")
|
raise RuntimeError(f"Failed to fetch page {paginated_url} after all attempts.")
|
||||||
|
|
||||||
|
|
||||||
def fetch_single_post_data(api_domain, service, user_id, post_id, headers, logger, cookies_dict=None):
|
def fetch_single_post_data(api_domain, service, user_id, post_id, headers, logger, cookies_dict=None):
|
||||||
"""
|
"""
|
||||||
--- MODIFIED FUNCTION ---
|
--- NEW FUNCTION ---
|
||||||
Fetches the full data, including the 'content' field, for a single post using cloudscraper.
|
Fetches the full data, including the 'content' field, for a single post.
|
||||||
"""
|
"""
|
||||||
post_api_url = f"https://{api_domain}/api/v1/{service}/user/{user_id}/post/{post_id}"
|
post_api_url = f"https://{api_domain}/api/v1/{service}/user/{user_id}/post/{post_id}"
|
||||||
logger(f" Fetching full content for post ID {post_id}...")
|
logger(f" Fetching full content for post ID {post_id}...")
|
||||||
|
|
||||||
scraper = cloudscraper.create_scraper()
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = scraper.get(post_api_url, headers=headers, timeout=(15, 300), cookies=cookies_dict)
|
with requests.get(post_api_url, headers=headers, timeout=(15, 300), cookies=cookies_dict, stream=True) as response:
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
|
response_body = b""
|
||||||
full_post_data = response.json()
|
for chunk in response.iter_content(chunk_size=8192):
|
||||||
|
response_body += chunk
|
||||||
if isinstance(full_post_data, list) and full_post_data:
|
|
||||||
return full_post_data[0]
|
full_post_data = json.loads(response_body)
|
||||||
if isinstance(full_post_data, dict) and 'post' in full_post_data:
|
if isinstance(full_post_data, list) and full_post_data:
|
||||||
return full_post_data['post']
|
return full_post_data[0]
|
||||||
return full_post_data
|
return full_post_data
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger(f" ❌ Failed to fetch full content for post {post_id}: {e}")
|
logger(f" ❌ Failed to fetch full content for post {post_id}: {e}")
|
||||||
return None
|
return None
|
||||||
@@ -117,7 +101,6 @@ def fetch_post_comments(api_domain, service, user_id, post_id, headers, logger,
|
|||||||
try:
|
try:
|
||||||
response = requests.get(comments_api_url, headers=headers, timeout=(10, 30), cookies=cookies_dict)
|
response = requests.get(comments_api_url, headers=headers, timeout=(10, 30), cookies=cookies_dict)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
response.encoding = 'utf-8'
|
|
||||||
return response.json()
|
return response.json()
|
||||||
except requests.exceptions.RequestException as e:
|
except requests.exceptions.RequestException as e:
|
||||||
raise RuntimeError(f"Error fetching comments for post {post_id}: {e}")
|
raise RuntimeError(f"Error fetching comments for post {post_id}: {e}")
|
||||||
@@ -139,16 +122,11 @@ def download_from_api(
|
|||||||
manga_filename_style_for_sort_check=None,
|
manga_filename_style_for_sort_check=None,
|
||||||
processed_post_ids=None,
|
processed_post_ids=None,
|
||||||
fetch_all_first=False
|
fetch_all_first=False
|
||||||
):
|
):
|
||||||
parsed_input_url_for_domain = urlparse(api_url_input)
|
|
||||||
api_domain = parsed_input_url_for_domain.netloc
|
|
||||||
|
|
||||||
headers = {
|
headers = {
|
||||||
'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
|
'User-Agent': 'Mozilla/5.0',
|
||||||
'Referer': f'https://{api_domain}/',
|
'Accept': 'application/json'
|
||||||
'Accept': 'text/css'
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if processed_post_ids is None:
|
if processed_post_ids is None:
|
||||||
processed_post_ids = set()
|
processed_post_ids = set()
|
||||||
else:
|
else:
|
||||||
@@ -160,11 +138,15 @@ def download_from_api(
|
|||||||
logger(" Download_from_api cancelled at start.")
|
logger(" Download_from_api cancelled at start.")
|
||||||
return
|
return
|
||||||
|
|
||||||
# The code that defined api_domain was moved from here to the top of the function
|
parsed_input_url_for_domain = urlparse(api_url_input)
|
||||||
|
api_domain = parsed_input_url_for_domain.netloc
|
||||||
|
|
||||||
|
# --- START: MODIFIED LOGIC ---
|
||||||
|
# This list is updated to include the new .cr and .st mirrors for validation.
|
||||||
if not any(d in api_domain.lower() for d in ['kemono.su', 'kemono.party', 'kemono.cr', 'coomer.su', 'coomer.party', 'coomer.st']):
|
if not any(d in api_domain.lower() for d in ['kemono.su', 'kemono.party', 'kemono.cr', 'coomer.su', 'coomer.party', 'coomer.st']):
|
||||||
logger(f"⚠️ Unrecognized domain '{api_domain}' from input URL. Defaulting to kemono.su for API calls.")
|
logger(f"⚠️ Unrecognized domain '{api_domain}' from input URL. Defaulting to kemono.su for API calls.")
|
||||||
api_domain = "kemono.su"
|
api_domain = "kemono.su"
|
||||||
|
# --- END: MODIFIED LOGIC ---
|
||||||
|
|
||||||
cookies_for_api = None
|
cookies_for_api = None
|
||||||
if use_cookie and app_base_dir:
|
if use_cookie and app_base_dir:
|
||||||
@@ -178,7 +160,6 @@ def download_from_api(
|
|||||||
try:
|
try:
|
||||||
direct_response = requests.get(direct_post_api_url, headers=headers, timeout=(10, 30), cookies=cookies_for_api)
|
direct_response = requests.get(direct_post_api_url, headers=headers, timeout=(10, 30), cookies=cookies_for_api)
|
||||||
direct_response.raise_for_status()
|
direct_response.raise_for_status()
|
||||||
direct_response.encoding = 'utf-8'
|
|
||||||
direct_post_data = direct_response.json()
|
direct_post_data = direct_response.json()
|
||||||
if isinstance(direct_post_data, list) and direct_post_data:
|
if isinstance(direct_post_data, list) and direct_post_data:
|
||||||
direct_post_data = direct_post_data[0]
|
direct_post_data = direct_post_data[0]
|
||||||
@@ -204,7 +185,7 @@ def download_from_api(
|
|||||||
|
|
||||||
is_manga_mode_fetch_all_and_sort_oldest_first = manga_mode and (manga_filename_style_for_sort_check != STYLE_DATE_POST_TITLE) and not target_post_id
|
is_manga_mode_fetch_all_and_sort_oldest_first = manga_mode and (manga_filename_style_for_sort_check != STYLE_DATE_POST_TITLE) and not target_post_id
|
||||||
should_fetch_all = fetch_all_first or is_manga_mode_fetch_all_and_sort_oldest_first
|
should_fetch_all = fetch_all_first or is_manga_mode_fetch_all_and_sort_oldest_first
|
||||||
api_base_url = f"https://{api_domain}/api/v1/{service}/user/{user_id}/posts"
|
api_base_url = f"https://{api_domain}/api/v1/{service}/user/{user_id}"
|
||||||
page_size = 50
|
page_size = 50
|
||||||
if is_manga_mode_fetch_all_and_sort_oldest_first:
|
if is_manga_mode_fetch_all_and_sort_oldest_first:
|
||||||
logger(f" Manga Mode (Style: {manga_filename_style_for_sort_check if manga_filename_style_for_sort_check else 'Default'} - Oldest First Sort Active): Fetching all posts to sort by date...")
|
logger(f" Manga Mode (Style: {manga_filename_style_for_sort_check if manga_filename_style_for_sort_check else 'Default'} - Oldest First Sort Active): Fetching all posts to sort by date...")
|
||||||
@@ -375,4 +356,3 @@ def download_from_api(
|
|||||||
time.sleep(0.6)
|
time.sleep(0.6)
|
||||||
if target_post_id and not processed_target_post_flag and not (cancellation_event and cancellation_event.is_set()):
|
if target_post_id and not processed_target_post_flag and not (cancellation_event and cancellation_event.is_set()):
|
||||||
logger(f"❌ Target post {target_post_id} could not be found after checking all relevant pages (final check after loop).")
|
logger(f"❌ Target post {target_post_id} could not be found after checking all relevant pages (final check after loop).")
|
||||||
|
|
||||||
|
|||||||
@@ -1,249 +0,0 @@
|
|||||||
import logging
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import requests
|
|
||||||
import html
|
|
||||||
import time
|
|
||||||
import datetime
|
|
||||||
import urllib.parse
|
|
||||||
import json
|
|
||||||
import random
|
|
||||||
import binascii
|
|
||||||
import itertools
|
|
||||||
|
|
||||||
class MockMessage:
|
|
||||||
Directory = 1
|
|
||||||
Url = 2
|
|
||||||
Version = 3
|
|
||||||
|
|
||||||
class AlbumException(Exception): pass
|
|
||||||
class ExtractionError(AlbumException): pass
|
|
||||||
class HttpError(ExtractionError):
|
|
||||||
def __init__(self, message="", response=None):
|
|
||||||
self.response = response
|
|
||||||
self.status = response.status_code if response is not None else 0
|
|
||||||
super().__init__(message)
|
|
||||||
class ControlException(AlbumException): pass
|
|
||||||
class AbortExtraction(ExtractionError, ControlException): pass
|
|
||||||
|
|
||||||
try:
|
|
||||||
re_compile = re._compiler.compile
|
|
||||||
except AttributeError:
|
|
||||||
re_compile = re.sre_compile.compile
|
|
||||||
HTML_RE = re_compile(r"<[^>]+>")
|
|
||||||
def extr(txt, begin, end, default=""):
|
|
||||||
try:
|
|
||||||
first = txt.index(begin) + len(begin)
|
|
||||||
return txt[first:txt.index(end, first)]
|
|
||||||
except Exception: return default
|
|
||||||
def extract_iter(txt, begin, end, pos=None):
|
|
||||||
try:
|
|
||||||
index = txt.index
|
|
||||||
lbeg = len(begin)
|
|
||||||
lend = len(end)
|
|
||||||
while True:
|
|
||||||
first = index(begin, pos) + lbeg
|
|
||||||
last = index(end, first)
|
|
||||||
pos = last + lend
|
|
||||||
yield txt[first:last]
|
|
||||||
except Exception: return
|
|
||||||
def split_html(txt):
|
|
||||||
try: return [html.unescape(x).strip() for x in HTML_RE.split(txt) if x and not x.isspace()]
|
|
||||||
except TypeError: return []
|
|
||||||
def parse_datetime(date_string, format="%Y-%m-%dT%H:%M:%S%z", utcoffset=0):
|
|
||||||
try:
|
|
||||||
d = datetime.datetime.strptime(date_string, format)
|
|
||||||
o = d.utcoffset()
|
|
||||||
if o is not None: d = d.replace(tzinfo=None, microsecond=0) - o
|
|
||||||
else:
|
|
||||||
if d.microsecond: d = d.replace(microsecond=0)
|
|
||||||
if utcoffset: d += datetime.timedelta(0, utcoffset * -3600)
|
|
||||||
return d
|
|
||||||
except (TypeError, IndexError, KeyError, ValueError, OverflowError): return None
|
|
||||||
unquote = urllib.parse.unquote
|
|
||||||
unescape = html.unescape
|
|
||||||
|
|
||||||
# --- From: util.py ---
|
|
||||||
def decrypt_xor(encrypted, key, base64=True, fromhex=False):
|
|
||||||
if base64: encrypted = binascii.a2b_base64(encrypted)
|
|
||||||
if fromhex: encrypted = bytes.fromhex(encrypted.decode())
|
|
||||||
div = len(key)
|
|
||||||
return bytes([encrypted[i] ^ key[i % div] for i in range(len(encrypted))]).decode()
|
|
||||||
def advance(iterable, num):
|
|
||||||
iterator = iter(iterable)
|
|
||||||
next(itertools.islice(iterator, num, num), None)
|
|
||||||
return iterator
|
|
||||||
def json_loads(s): return json.loads(s)
|
|
||||||
def json_dumps(obj): return json.dumps(obj, separators=(",", ":"))
|
|
||||||
|
|
||||||
# --- From: common.py ---
|
|
||||||
class Extractor:
|
|
||||||
def __init__(self, match, logger):
|
|
||||||
self.log = logger
|
|
||||||
self.url = match.string
|
|
||||||
self.match = match
|
|
||||||
self.groups = match.groups()
|
|
||||||
self.session = requests.Session()
|
|
||||||
self.session.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0"
|
|
||||||
@classmethod
|
|
||||||
def from_url(cls, url, logger):
|
|
||||||
if isinstance(cls.pattern, str): cls.pattern = re.compile(cls.pattern)
|
|
||||||
match = cls.pattern.match(url)
|
|
||||||
return cls(match, logger) if match else None
|
|
||||||
def __iter__(self): return self.items()
|
|
||||||
def items(self): yield MockMessage.Version, 1
|
|
||||||
def request(self, url, method="GET", fatal=True, **kwargs):
|
|
||||||
tries = 1
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
response = self.session.request(method, url, **kwargs)
|
|
||||||
if response.status_code < 400: return response
|
|
||||||
msg = f"'{response.status_code} {response.reason}' for '{response.url}'"
|
|
||||||
except requests.exceptions.RequestException as exc:
|
|
||||||
msg = str(exc)
|
|
||||||
|
|
||||||
self.log.info("%s (retrying...)", msg)
|
|
||||||
if tries > 4: break
|
|
||||||
time.sleep(tries)
|
|
||||||
tries += 1
|
|
||||||
if not fatal: return None
|
|
||||||
raise HttpError(msg)
|
|
||||||
def request_json(self, url, **kwargs):
|
|
||||||
response = self.request(url, **kwargs)
|
|
||||||
try: return json_loads(response.text)
|
|
||||||
except Exception as exc:
|
|
||||||
self.log.warning("%s: %s", exc.__class__.__name__, exc)
|
|
||||||
if not kwargs.get("fatal", True): return {}
|
|
||||||
raise
|
|
||||||
|
|
||||||
# --- From: bunkr.py (Adapted) ---
|
|
||||||
BASE_PATTERN_BUNKR = r"(?:https?://)?(?:[a-zA-Z0-9-]+\.)?(bunkr\.(?:si|la|ws|red|black|media|site|is|to|ac|cr|ci|fi|pk|ps|sk|ph|su)|bunkrr\.ru)"
|
|
||||||
DOMAINS = ["bunkr.si", "bunkr.ws", "bunkr.la", "bunkr.red", "bunkr.black", "bunkr.media", "bunkr.site"]
|
|
||||||
CF_DOMAINS = set()
|
|
||||||
|
|
||||||
class BunkrAlbumExtractor(Extractor):
|
|
||||||
category = "bunkr"
|
|
||||||
root = "https://bunkr.si"
|
|
||||||
root_dl = "https://get.bunkrr.su"
|
|
||||||
root_api = "https://apidl.bunkr.ru"
|
|
||||||
pattern = re.compile(BASE_PATTERN_BUNKR + r"/a/([^/?#]+)")
|
|
||||||
|
|
||||||
def __init__(self, match, logger):
|
|
||||||
super().__init__(match, logger)
|
|
||||||
domain_match = re.search(BASE_PATTERN_BUNKR, match.string)
|
|
||||||
if domain_match:
|
|
||||||
self.root = "https://" + domain_match.group(1)
|
|
||||||
self.endpoint = self.root_api + "/api/_001_v2"
|
|
||||||
self.album_id = self.groups[-1]
|
|
||||||
|
|
||||||
def items(self):
|
|
||||||
page = self.request(self.url).text
|
|
||||||
title = unescape(unescape(extr(page, 'property="og:title" content="', '"')))
|
|
||||||
items_html = list(extract_iter(page, '<div class="grid-images_box', "</a>"))
|
|
||||||
|
|
||||||
album_data = {
|
|
||||||
"album_id": self.album_id,
|
|
||||||
"album_name": title,
|
|
||||||
"count": len(items_html),
|
|
||||||
}
|
|
||||||
yield MockMessage.Directory, album_data, {}
|
|
||||||
|
|
||||||
for item_html in items_html:
|
|
||||||
try:
|
|
||||||
webpage_url = unescape(extr(item_html, ' href="', '"'))
|
|
||||||
if webpage_url.startswith("/"):
|
|
||||||
webpage_url = self.root + webpage_url
|
|
||||||
|
|
||||||
file_data = self._extract_file(webpage_url)
|
|
||||||
info = split_html(item_html)
|
|
||||||
|
|
||||||
if not file_data.get("name"):
|
|
||||||
file_data["name"] = info[-3]
|
|
||||||
|
|
||||||
yield MockMessage.Url, file_data, {}
|
|
||||||
except Exception as exc:
|
|
||||||
self.log.error("%s: %s", exc.__class__.__name__, exc)
|
|
||||||
|
|
||||||
def _extract_file(self, webpage_url):
|
|
||||||
page = self.request(webpage_url).text
|
|
||||||
data_id = extr(page, 'data-file-id="', '"')
|
|
||||||
referer = self.root_dl + "/file/" + data_id
|
|
||||||
headers = {"Referer": referer, "Origin": self.root_dl}
|
|
||||||
data = self.request_json(self.endpoint, method="POST", headers=headers, json={"id": data_id})
|
|
||||||
|
|
||||||
file_url = decrypt_xor(data["url"], f"SECRET_KEY_{data['timestamp'] // 3600}".encode()) if data.get("encrypted") else data["url"]
|
|
||||||
file_name = extr(page, "<h1", "<").rpartition(">")[2]
|
|
||||||
|
|
||||||
return {
|
|
||||||
"url": file_url,
|
|
||||||
"name": unescape(file_name),
|
|
||||||
"_http_headers": {"Referer": referer}
|
|
||||||
}
|
|
||||||
|
|
||||||
class BunkrMediaExtractor(BunkrAlbumExtractor):
|
|
||||||
pattern = re.compile(BASE_PATTERN_BUNKR + r"(/[fvid]/[^/?#]+)")
|
|
||||||
def items(self):
|
|
||||||
try:
|
|
||||||
media_path = self.groups[-1]
|
|
||||||
file_data = self._extract_file(self.root + media_path)
|
|
||||||
album_data = {"album_name": file_data.get("name", "bunkr_media"), "count": 1}
|
|
||||||
|
|
||||||
yield MockMessage.Directory, album_data, {}
|
|
||||||
yield MockMessage.Url, file_data, {}
|
|
||||||
|
|
||||||
except Exception as exc:
|
|
||||||
self.log.error("%s: %s", exc.__class__.__name__, exc)
|
|
||||||
yield MockMessage.Directory, {"album_name": "error", "count": 0}, {}
|
|
||||||
|
|
||||||
# ==============================================================================
|
|
||||||
# --- PUBLIC API FOR THE GUI ---
|
|
||||||
# ==============================================================================
|
|
||||||
|
|
||||||
def get_bunkr_extractor(url, logger):
|
|
||||||
"""Selects the correct Bunkr extractor based on the URL pattern."""
|
|
||||||
if BunkrAlbumExtractor.pattern.match(url):
|
|
||||||
logger.info("Bunkr Album URL detected.")
|
|
||||||
return BunkrAlbumExtractor.from_url(url, logger)
|
|
||||||
elif BunkrMediaExtractor.pattern.match(url):
|
|
||||||
logger.info("Bunkr Media URL detected.")
|
|
||||||
return BunkrMediaExtractor.from_url(url, logger)
|
|
||||||
else:
|
|
||||||
logger.error(f"No suitable Bunkr extractor found for URL: {url}")
|
|
||||||
return None
|
|
||||||
|
|
||||||
def fetch_bunkr_data(url, logger):
|
|
||||||
"""
|
|
||||||
Main function to be called from the GUI.
|
|
||||||
It extracts all file information from a Bunkr URL.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
A tuple of (album_name, list_of_files)
|
|
||||||
- album_name (str): The name of the album.
|
|
||||||
- list_of_files (list): A list of dicts, each containing 'url', 'name', and '_http_headers'.
|
|
||||||
Returns (None, None) on failure.
|
|
||||||
"""
|
|
||||||
extractor = get_bunkr_extractor(url, logger)
|
|
||||||
if not extractor:
|
|
||||||
return None, None
|
|
||||||
|
|
||||||
try:
|
|
||||||
album_name = "default_bunkr_album"
|
|
||||||
files_to_download = []
|
|
||||||
for msg_type, data, metadata in extractor:
|
|
||||||
if msg_type == MockMessage.Directory:
|
|
||||||
raw_album_name = data.get('album_name', 'untitled')
|
|
||||||
album_name = re.sub(r'[<>:"/\\|?*]', '_', raw_album_name).strip() or "untitled"
|
|
||||||
logger.info(f"Processing Bunkr album: {album_name}")
|
|
||||||
elif msg_type == MockMessage.Url:
|
|
||||||
# data here is the file_data dictionary
|
|
||||||
files_to_download.append(data)
|
|
||||||
|
|
||||||
if not files_to_download:
|
|
||||||
logger.warning("No files found to download from the Bunkr URL.")
|
|
||||||
return None, None
|
|
||||||
|
|
||||||
return album_name, files_to_download
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"An error occurred while extracting Bunkr info: {e}", exc_info=True)
|
|
||||||
return None, None
|
|
||||||
@@ -1,70 +1,63 @@
|
|||||||
import time
|
import time
|
||||||
import cloudscraper
|
import requests
|
||||||
import json
|
import json
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
def fetch_server_channels(server_id, logger=print, cookies_dict=None):
|
def fetch_server_channels(server_id, logger, cookies=None, cancellation_event=None, pause_event=None):
|
||||||
"""
|
"""
|
||||||
Fetches all channels for a given Discord server ID from the API.
|
Fetches the list of channels for a given Discord server ID from the Kemono API.
|
||||||
Uses cloudscraper to bypass Cloudflare.
|
UPDATED to be pausable and cancellable.
|
||||||
"""
|
"""
|
||||||
api_url = f"https://kemono.cr/api/v1/discord/server/{server_id}"
|
domains_to_try = ["kemono.cr", "kemono.su"]
|
||||||
logger(f" Fetching channels for server: {api_url}")
|
for domain in domains_to_try:
|
||||||
|
|
||||||
scraper = cloudscraper.create_scraper()
|
|
||||||
headers = {
|
|
||||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
|
||||||
'Referer': f'https://kemono.cr/discord/server/{server_id}',
|
|
||||||
'Accept': 'text/css'
|
|
||||||
}
|
|
||||||
|
|
||||||
try:
|
|
||||||
response = scraper.get(api_url, headers=headers, cookies=cookies_dict, timeout=30)
|
|
||||||
response.raise_for_status()
|
|
||||||
channels = response.json()
|
|
||||||
if isinstance(channels, list):
|
|
||||||
logger(f" ✅ Found {len(channels)} channels for server {server_id}.")
|
|
||||||
return channels
|
|
||||||
return None
|
|
||||||
except Exception as e:
|
|
||||||
logger(f" ❌ Error fetching server channels for {server_id}: {e}")
|
|
||||||
return None
|
|
||||||
|
|
||||||
def fetch_channel_messages(channel_id, logger=print, cancellation_event=None, pause_event=None, cookies_dict=None):
|
|
||||||
"""
|
|
||||||
A generator that fetches all messages for a specific Discord channel, handling pagination.
|
|
||||||
Uses cloudscraper and proper headers to bypass server protection.
|
|
||||||
"""
|
|
||||||
scraper = cloudscraper.create_scraper()
|
|
||||||
base_url = f"https://kemono.cr/api/v1/discord/channel/{channel_id}"
|
|
||||||
headers = {
|
|
||||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
|
||||||
'Referer': f'https://kemono.cr/discord/channel/{channel_id}',
|
|
||||||
'Accept': 'text/css'
|
|
||||||
}
|
|
||||||
|
|
||||||
offset = 0
|
|
||||||
# --- FIX: Corrected the page size for Discord API pagination ---
|
|
||||||
page_size = 150
|
|
||||||
# --- END FIX ---
|
|
||||||
|
|
||||||
while True:
|
|
||||||
if cancellation_event and cancellation_event.is_set():
|
if cancellation_event and cancellation_event.is_set():
|
||||||
logger(" Discord message fetching cancelled.")
|
logger(" Channel fetching cancelled by user.")
|
||||||
break
|
return None
|
||||||
if pause_event and pause_event.is_set():
|
while pause_event and pause_event.is_set():
|
||||||
logger(" Discord message fetching paused...")
|
if cancellation_event and cancellation_event.is_set(): break
|
||||||
while pause_event.is_set():
|
time.sleep(0.5)
|
||||||
if cancellation_event and cancellation_event.is_set():
|
|
||||||
break
|
|
||||||
time.sleep(0.5)
|
|
||||||
if not (cancellation_event and cancellation_event.is_set()):
|
|
||||||
logger(" Discord message fetching resumed.")
|
|
||||||
|
|
||||||
paginated_url = f"{base_url}?o={offset}"
|
lookup_url = f"https://{domain}/api/v1/discord/channel/lookup/{server_id}"
|
||||||
|
logger(f" Attempting to fetch channel list from: {lookup_url}")
|
||||||
|
try:
|
||||||
|
response = requests.get(lookup_url, cookies=cookies, timeout=15)
|
||||||
|
response.raise_for_status()
|
||||||
|
channels = response.json()
|
||||||
|
if isinstance(channels, list):
|
||||||
|
logger(f" ✅ Found {len(channels)} channels for server {server_id}.")
|
||||||
|
return channels
|
||||||
|
except (requests.exceptions.RequestException, json.JSONDecodeError):
|
||||||
|
# This is a silent failure, we'll just try the next domain
|
||||||
|
pass
|
||||||
|
|
||||||
|
logger(f" ❌ Failed to fetch channel list for server {server_id} from all available domains.")
|
||||||
|
return None
|
||||||
|
|
||||||
|
def fetch_channel_messages(channel_id, logger, cancellation_event, pause_event, cookies=None):
|
||||||
|
"""
|
||||||
|
Fetches all messages from a Discord channel by looping through API pages (pagination).
|
||||||
|
Uses a page size of 150 and handles the specific offset logic.
|
||||||
|
"""
|
||||||
|
offset = 0
|
||||||
|
page_size = 150 # Corrected page size based on your findings
|
||||||
|
api_base_url = f"https://kemono.cr/api/v1/discord/channel/{channel_id}"
|
||||||
|
|
||||||
|
while not (cancellation_event and cancellation_event.is_set()):
|
||||||
|
if pause_event and pause_event.is_set():
|
||||||
|
logger(" Message fetching paused...")
|
||||||
|
while pause_event.is_set():
|
||||||
|
if cancellation_event and cancellation_event.is_set(): break
|
||||||
|
time.sleep(0.5)
|
||||||
|
logger(" Message fetching resumed.")
|
||||||
|
|
||||||
|
if cancellation_event and cancellation_event.is_set():
|
||||||
|
break
|
||||||
|
|
||||||
|
paginated_url = f"{api_base_url}?o={offset}"
|
||||||
logger(f" Fetching messages from API: page starting at offset {offset}")
|
logger(f" Fetching messages from API: page starting at offset {offset}")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
response = scraper.get(paginated_url, headers=headers, cookies=cookies_dict, timeout=30)
|
response = requests.get(paginated_url, cookies=cookies, timeout=20)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
messages_batch = response.json()
|
messages_batch = response.json()
|
||||||
|
|
||||||
@@ -80,11 +73,8 @@ def fetch_channel_messages(channel_id, logger=print, cancellation_event=None, pa
|
|||||||
break
|
break
|
||||||
|
|
||||||
offset += page_size
|
offset += page_size
|
||||||
time.sleep(0.5) # Be respectful to the API
|
time.sleep(0.5)
|
||||||
|
|
||||||
except (cloudscraper.exceptions.CloudflareException, json.JSONDecodeError) as e:
|
except (requests.exceptions.RequestException, json.JSONDecodeError) as e:
|
||||||
logger(f" ❌ Error fetching messages at offset {offset}: {e}")
|
logger(f" ❌ Error fetching messages at offset {offset}: {e}")
|
||||||
break
|
break
|
||||||
except Exception as e:
|
|
||||||
logger(f" ❌ An unexpected error occurred while fetching messages: {e}")
|
|
||||||
break
|
|
||||||
@@ -1,147 +0,0 @@
|
|||||||
# src/core/erome_client.py
|
|
||||||
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import html
|
|
||||||
import time
|
|
||||||
import urllib.parse
|
|
||||||
import requests
|
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
# #############################################################################
|
|
||||||
# SECTION: Utility functions adapted from the original script
|
|
||||||
# #############################################################################
|
|
||||||
|
|
||||||
def extr(txt, begin, end, default=""):
|
|
||||||
"""Stripped-down version of 'extract()' to find text between two delimiters."""
|
|
||||||
try:
|
|
||||||
first = txt.index(begin) + len(begin)
|
|
||||||
return txt[first:txt.index(end, first)]
|
|
||||||
except (ValueError, IndexError):
|
|
||||||
return default
|
|
||||||
|
|
||||||
def extract_iter(txt, begin, end):
|
|
||||||
"""Yields all occurrences of text between two delimiters."""
|
|
||||||
try:
|
|
||||||
index = txt.index
|
|
||||||
lbeg = len(begin)
|
|
||||||
lend = len(end)
|
|
||||||
pos = 0
|
|
||||||
while True:
|
|
||||||
first = index(begin, pos) + lbeg
|
|
||||||
last = index(end, first)
|
|
||||||
pos = last + lend
|
|
||||||
yield txt[first:last]
|
|
||||||
except (ValueError, IndexError):
|
|
||||||
return
|
|
||||||
|
|
||||||
def nameext_from_url(url):
|
|
||||||
"""Extracts filename and extension from a URL."""
|
|
||||||
data = {}
|
|
||||||
filename = urllib.parse.unquote(url.partition("?")[0].rpartition("/")[2])
|
|
||||||
name, _, ext = filename.rpartition(".")
|
|
||||||
if name and len(ext) <= 16:
|
|
||||||
data["filename"], data["extension"] = name, ext.lower()
|
|
||||||
else:
|
|
||||||
data["filename"], data["extension"] = filename, ""
|
|
||||||
return data
|
|
||||||
|
|
||||||
def parse_timestamp(ts, default=None):
|
|
||||||
"""Creates a datetime object from a Unix timestamp."""
|
|
||||||
try:
|
|
||||||
# Use fromtimestamp for simplicity and compatibility
|
|
||||||
return datetime.fromtimestamp(int(ts))
|
|
||||||
except (ValueError, TypeError):
|
|
||||||
return default
|
|
||||||
|
|
||||||
# #############################################################################
|
|
||||||
# SECTION: Main Erome Fetching Logic
|
|
||||||
# #############################################################################
|
|
||||||
|
|
||||||
def fetch_erome_data(url, logger):
|
|
||||||
"""
|
|
||||||
Identifies and extracts all media files from an Erome album URL.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
url (str): The Erome album URL (e.g., https://www.erome.com/a/albumID).
|
|
||||||
logger (function): A function to log progress messages.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
tuple: A tuple containing (album_folder_name, list_of_file_dicts).
|
|
||||||
Returns (None, []) if data extraction fails.
|
|
||||||
"""
|
|
||||||
album_id_match = re.search(r"/a/(\w+)", url)
|
|
||||||
if not album_id_match:
|
|
||||||
logger(f"Error: The URL '{url}' does not appear to be a valid Erome album link.")
|
|
||||||
return None, []
|
|
||||||
|
|
||||||
album_id = album_id_match.group(1)
|
|
||||||
page_url = f"https://www.erome.com/a/{album_id}"
|
|
||||||
|
|
||||||
session = requests.Session()
|
|
||||||
session.headers.update({
|
|
||||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
|
|
||||||
"Referer": "https://www.erome.com/"
|
|
||||||
})
|
|
||||||
|
|
||||||
try:
|
|
||||||
logger(f" Fetching Erome album page: {page_url}")
|
|
||||||
# Add a loop to handle "Please wait" pages
|
|
||||||
for attempt in range(5):
|
|
||||||
response = session.get(page_url, timeout=30)
|
|
||||||
response.raise_for_status()
|
|
||||||
page_content = response.text
|
|
||||||
if "<title>Please wait a few moments</title>" in page_content:
|
|
||||||
logger(f" Cloudflare check detected. Waiting 5 seconds... (Attempt {attempt + 1}/5)")
|
|
||||||
time.sleep(5)
|
|
||||||
continue
|
|
||||||
break
|
|
||||||
else:
|
|
||||||
logger(" Error: Could not bypass Cloudflare check after several attempts.")
|
|
||||||
return None, []
|
|
||||||
|
|
||||||
title = html.unescape(extr(page_content, 'property="og:title" content="', '"'))
|
|
||||||
user = urllib.parse.unquote(extr(page_content, 'href="https://www.erome.com/', '"', default="unknown_user"))
|
|
||||||
|
|
||||||
# Sanitize title and user for folder creation
|
|
||||||
sanitized_title = re.sub(r'[<>:"/\\|?*]', '_', title).strip()
|
|
||||||
sanitized_user = re.sub(r'[<>:"/\\|?*]', '_', user).strip()
|
|
||||||
|
|
||||||
album_folder_name = f"Erome - {sanitized_user} - {sanitized_title} [{album_id}]"
|
|
||||||
|
|
||||||
urls = []
|
|
||||||
# Split the page content by media groups to find all videos
|
|
||||||
media_groups = page_content.split('<div class="media-group"')
|
|
||||||
for group in media_groups[1:]: # Skip the part before the first media group
|
|
||||||
# Prioritize <source> tag, fall back to data-src for images
|
|
||||||
video_url = extr(group, '<source src="', '"') or extr(group, 'data-src="', '"')
|
|
||||||
if video_url:
|
|
||||||
urls.append(video_url)
|
|
||||||
|
|
||||||
if not urls:
|
|
||||||
logger(" Warning: No media URLs found on the album page.")
|
|
||||||
return album_folder_name, []
|
|
||||||
|
|
||||||
logger(f" Found {len(urls)} media files in album '{title}'.")
|
|
||||||
|
|
||||||
file_list = []
|
|
||||||
for i, file_url in enumerate(urls, 1):
|
|
||||||
filename_info = nameext_from_url(file_url)
|
|
||||||
# Create a clean, descriptive filename
|
|
||||||
filename = f"{album_id}_{sanitized_title}_{i:03d}.{filename_info.get('extension', 'mp4')}"
|
|
||||||
|
|
||||||
file_data = {
|
|
||||||
"url": file_url,
|
|
||||||
"filename": filename,
|
|
||||||
"headers": {"Referer": page_url},
|
|
||||||
}
|
|
||||||
file_list.append(file_data)
|
|
||||||
|
|
||||||
return album_folder_name, file_list
|
|
||||||
|
|
||||||
except requests.exceptions.RequestException as e:
|
|
||||||
logger(f" Error fetching Erome page: {e}")
|
|
||||||
return None, []
|
|
||||||
except Exception as e:
|
|
||||||
logger(f" An unexpected error occurred during Erome extraction: {e}")
|
|
||||||
return None, []
|
|
||||||
@@ -1,45 +0,0 @@
|
|||||||
import requests
|
|
||||||
import cloudscraper
|
|
||||||
import json
|
|
||||||
|
|
||||||
def fetch_nhentai_gallery(gallery_id, logger=print):
|
|
||||||
"""
|
|
||||||
Fetches the metadata for a single nhentai gallery using cloudscraper to bypass Cloudflare.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
gallery_id (str or int): The ID of the nhentai gallery.
|
|
||||||
logger (function): A function to log progress and error messages.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
dict: A dictionary containing the gallery's metadata if successful, otherwise None.
|
|
||||||
"""
|
|
||||||
api_url = f"https://nhentai.net/api/gallery/{gallery_id}"
|
|
||||||
|
|
||||||
# Create a cloudscraper instance
|
|
||||||
scraper = cloudscraper.create_scraper()
|
|
||||||
|
|
||||||
logger(f" Fetching nhentai gallery metadata from: {api_url}")
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Use the scraper to make the GET request
|
|
||||||
response = scraper.get(api_url, timeout=20)
|
|
||||||
|
|
||||||
if response.status_code == 404:
|
|
||||||
logger(f" ❌ Gallery not found (404): ID {gallery_id}")
|
|
||||||
return None
|
|
||||||
|
|
||||||
response.raise_for_status()
|
|
||||||
|
|
||||||
gallery_data = response.json()
|
|
||||||
|
|
||||||
if "id" in gallery_data and "media_id" in gallery_data and "images" in gallery_data:
|
|
||||||
logger(f" ✅ Successfully fetched metadata for '{gallery_data['title']['english']}'")
|
|
||||||
gallery_data['pages'] = gallery_data.pop('images')['pages']
|
|
||||||
return gallery_data
|
|
||||||
else:
|
|
||||||
logger(" ❌ API response is missing essential keys (id, media_id, or images).")
|
|
||||||
return None
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger(f" ❌ An error occurred while fetching gallery {gallery_id}: {e}")
|
|
||||||
return None
|
|
||||||
@@ -1,173 +0,0 @@
|
|||||||
# src/core/saint2_client.py
|
|
||||||
|
|
||||||
import os
|
|
||||||
import re as re_module
|
|
||||||
import html
|
|
||||||
import urllib.parse
|
|
||||||
import requests
|
|
||||||
|
|
||||||
# ##############################################################################
|
|
||||||
# SECTION: Utility functions adapted from the original script
|
|
||||||
# ##############################################################################
|
|
||||||
|
|
||||||
PATTERN_CACHE = {}
|
|
||||||
|
|
||||||
def re(pattern):
|
|
||||||
"""Compile a regular expression pattern and cache it."""
|
|
||||||
try:
|
|
||||||
return PATTERN_CACHE[pattern]
|
|
||||||
except KeyError:
|
|
||||||
p = PATTERN_CACHE[pattern] = re_module.compile(pattern)
|
|
||||||
return p
|
|
||||||
|
|
||||||
def extract_from(txt, pos=None, default=""):
|
|
||||||
"""Returns a function that extracts text between two delimiters from 'txt'."""
|
|
||||||
def extr(begin, end, index=txt.find, txt=txt):
|
|
||||||
nonlocal pos
|
|
||||||
try:
|
|
||||||
start_pos = pos if pos is not None else 0
|
|
||||||
first = index(begin, start_pos) + len(begin)
|
|
||||||
last = index(end, first)
|
|
||||||
if pos is not None:
|
|
||||||
pos = last + len(end)
|
|
||||||
return txt[first:last]
|
|
||||||
except (ValueError, IndexError):
|
|
||||||
return default
|
|
||||||
return extr
|
|
||||||
|
|
||||||
def nameext_from_url(url):
|
|
||||||
"""Extract filename and extension from a URL."""
|
|
||||||
data = {}
|
|
||||||
filename = urllib.parse.unquote(url.partition("?")[0].rpartition("/")[2])
|
|
||||||
name, _, ext = filename.rpartition(".")
|
|
||||||
if name and len(ext) <= 16:
|
|
||||||
data["filename"], data["extension"] = name, ext.lower()
|
|
||||||
else:
|
|
||||||
data["filename"], data["extension"] = filename, ""
|
|
||||||
return data
|
|
||||||
|
|
||||||
# ##############################################################################
|
|
||||||
# SECTION: Extractor Logic adapted for the main application
|
|
||||||
# ##############################################################################
|
|
||||||
|
|
||||||
class BaseExtractor:
|
|
||||||
"""A simplified base class for extractors."""
|
|
||||||
def __init__(self, match, session, logger):
|
|
||||||
self.match = match
|
|
||||||
self.groups = match.groups()
|
|
||||||
self.session = session
|
|
||||||
self.log = logger
|
|
||||||
|
|
||||||
def request(self, url, **kwargs):
|
|
||||||
"""Makes an HTTP request using the session."""
|
|
||||||
try:
|
|
||||||
response = self.session.get(url, **kwargs)
|
|
||||||
response.raise_for_status()
|
|
||||||
return response
|
|
||||||
except requests.exceptions.RequestException as e:
|
|
||||||
self.log(f"Error making request to {url}: {e}")
|
|
||||||
return None
|
|
||||||
|
|
||||||
class SaintAlbumExtractor(BaseExtractor):
|
|
||||||
"""Extractor for saint.su albums."""
|
|
||||||
root = "https://saint2.su"
|
|
||||||
pattern = re(r"(?:https?://)?saint\d*\.(?:su|pk|cr|to)/a/([^/?#]+)")
|
|
||||||
|
|
||||||
def items(self):
|
|
||||||
"""Generator that yields all files from an album."""
|
|
||||||
album_id = self.groups[0]
|
|
||||||
response = self.request(f"{self.root}/a/{album_id}")
|
|
||||||
if not response:
|
|
||||||
return None, []
|
|
||||||
|
|
||||||
extr = extract_from(response.text)
|
|
||||||
title = extr("<title>", "<").rpartition(" - ")[0]
|
|
||||||
self.log(f"Downloading album: {title}")
|
|
||||||
|
|
||||||
files_html = re_module.findall(r'<a class="image".*?</a>', response.text, re_module.DOTALL)
|
|
||||||
file_list = []
|
|
||||||
for i, file_html in enumerate(files_html, 1):
|
|
||||||
file_extr = extract_from(file_html)
|
|
||||||
file_url = html.unescape(file_extr("onclick=\"play('", "'"))
|
|
||||||
if not file_url:
|
|
||||||
continue
|
|
||||||
|
|
||||||
filename_info = nameext_from_url(file_url)
|
|
||||||
filename = f"{filename_info['filename']}.{filename_info['extension']}"
|
|
||||||
|
|
||||||
file_data = {
|
|
||||||
"url": file_url,
|
|
||||||
"filename": filename,
|
|
||||||
"headers": {"Referer": response.url},
|
|
||||||
}
|
|
||||||
file_list.append(file_data)
|
|
||||||
|
|
||||||
return title, file_list
|
|
||||||
|
|
||||||
class SaintMediaExtractor(BaseExtractor):
|
|
||||||
"""Extractor for single saint.su media links."""
|
|
||||||
root = "https://saint2.su"
|
|
||||||
pattern = re(r"(?:https?://)?saint\d*\.(?:su|pk|cr|to)(/(embe)?d/([^/?#]+))")
|
|
||||||
|
|
||||||
def items(self):
|
|
||||||
"""Generator that yields the single file from a media page."""
|
|
||||||
path, embed, media_id = self.groups
|
|
||||||
url = self.root + path
|
|
||||||
response = self.request(url)
|
|
||||||
if not response:
|
|
||||||
return None, []
|
|
||||||
|
|
||||||
extr = extract_from(response.text)
|
|
||||||
file_url = ""
|
|
||||||
title = extr("<title>", "<").rpartition(" - ")[0] or media_id
|
|
||||||
|
|
||||||
if embed: # /embed/ link
|
|
||||||
file_url = html.unescape(extr('<source src="', '"'))
|
|
||||||
else: # /d/ link
|
|
||||||
file_url = html.unescape(extr('<a href="', '"'))
|
|
||||||
|
|
||||||
if not file_url:
|
|
||||||
self.log("Could not find video URL on the page.")
|
|
||||||
return title, []
|
|
||||||
|
|
||||||
filename_info = nameext_from_url(file_url)
|
|
||||||
filename = f"{filename_info['filename'] or media_id}.{filename_info['extension'] or 'mp4'}"
|
|
||||||
|
|
||||||
file_data = {
|
|
||||||
"url": file_url,
|
|
||||||
"filename": filename,
|
|
||||||
"headers": {"Referer": response.url}
|
|
||||||
}
|
|
||||||
|
|
||||||
return title, [file_data]
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_saint2_data(url, logger):
|
|
||||||
"""
|
|
||||||
Identifies the correct extractor for a saint2.su URL and returns the data.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
url (str): The saint2.su URL.
|
|
||||||
logger (function): A function to log progress messages.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
tuple: A tuple containing (album_title, list_of_file_dicts).
|
|
||||||
Returns (None, []) if no data could be fetched.
|
|
||||||
"""
|
|
||||||
extractors = [SaintMediaExtractor, SaintAlbumExtractor]
|
|
||||||
session = requests.Session()
|
|
||||||
session.headers.update({
|
|
||||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
|
|
||||||
})
|
|
||||||
|
|
||||||
for extractor_cls in extractors:
|
|
||||||
match = extractor_cls.pattern.match(url)
|
|
||||||
if match:
|
|
||||||
extractor = extractor_cls(match, session, logger)
|
|
||||||
album_title, files = extractor.items()
|
|
||||||
# Sanitize the album title to be a valid folder name
|
|
||||||
sanitized_title = re_module.sub(r'[<>:"/\\|?*]', '_', album_title) if album_title else "saint2_download"
|
|
||||||
return sanitized_title, files
|
|
||||||
|
|
||||||
logger(f"Error: The URL '{url}' does not match a known saint2 pattern.")
|
|
||||||
return None, []
|
|
||||||
@@ -15,8 +15,6 @@ from concurrent.futures import ThreadPoolExecutor, as_completed, CancelledError,
|
|||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
from urllib .parse import urlparse
|
from urllib .parse import urlparse
|
||||||
import requests
|
import requests
|
||||||
import cloudscraper
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
except ImportError:
|
except ImportError:
|
||||||
@@ -39,7 +37,7 @@ try:
|
|||||||
except ImportError:
|
except ImportError:
|
||||||
Document = None
|
Document = None
|
||||||
from PyQt5 .QtCore import Qt ,QThread ,pyqtSignal ,QMutex ,QMutexLocker ,QObject ,QTimer ,QSettings ,QStandardPaths ,QCoreApplication ,QUrl ,QSize ,QProcess
|
from PyQt5 .QtCore import Qt ,QThread ,pyqtSignal ,QMutex ,QMutexLocker ,QObject ,QTimer ,QSettings ,QStandardPaths ,QCoreApplication ,QUrl ,QSize ,QProcess
|
||||||
from .api_client import download_from_api, fetch_post_comments, fetch_single_post_data
|
from .api_client import download_from_api, fetch_post_comments
|
||||||
from ..services.multipart_downloader import download_file_in_parts, MULTIPART_DOWNLOADER_AVAILABLE
|
from ..services.multipart_downloader import download_file_in_parts, MULTIPART_DOWNLOADER_AVAILABLE
|
||||||
from ..services.drive_downloader import (
|
from ..services.drive_downloader import (
|
||||||
download_mega_file, download_gdrive_file, download_dropbox_file
|
download_mega_file, download_gdrive_file, download_dropbox_file
|
||||||
@@ -60,13 +58,18 @@ def robust_clean_name(name):
|
|||||||
"""A more robust function to remove illegal characters for filenames and folders."""
|
"""A more robust function to remove illegal characters for filenames and folders."""
|
||||||
if not name:
|
if not name:
|
||||||
return ""
|
return ""
|
||||||
illegal_chars_pattern = r'[\x00-\x1f<>:"/\\|?*\']'
|
# Removes illegal characters for Windows, macOS, and Linux: < > : " / \ | ? *
|
||||||
|
# Also removes control characters (ASCII 0-31) which are invisible but invalid.
|
||||||
|
illegal_chars_pattern = r'[\x00-\x1f<>:"/\\|?*]'
|
||||||
cleaned_name = re.sub(illegal_chars_pattern, '', name)
|
cleaned_name = re.sub(illegal_chars_pattern, '', name)
|
||||||
|
|
||||||
|
# Remove leading/trailing spaces or periods, which can cause issues.
|
||||||
cleaned_name = cleaned_name.strip(' .')
|
cleaned_name = cleaned_name.strip(' .')
|
||||||
|
|
||||||
|
# If the name is empty after cleaning (e.g., it was only illegal chars),
|
||||||
|
# provide a safe fallback name.
|
||||||
if not cleaned_name:
|
if not cleaned_name:
|
||||||
return "untitled_folder"
|
return "untitled_folder" # Or "untitled_file" depending on context
|
||||||
return cleaned_name
|
return cleaned_name
|
||||||
|
|
||||||
class PostProcessorSignals (QObject ):
|
class PostProcessorSignals (QObject ):
|
||||||
@@ -121,8 +124,7 @@ class PostProcessorWorker:
|
|||||||
processed_post_ids=None,
|
processed_post_ids=None,
|
||||||
multipart_scope='both',
|
multipart_scope='both',
|
||||||
multipart_parts_count=4,
|
multipart_parts_count=4,
|
||||||
multipart_min_size_mb=100,
|
multipart_min_size_mb=100
|
||||||
skip_file_size_mb=None
|
|
||||||
):
|
):
|
||||||
self.post = post_data
|
self.post = post_data
|
||||||
self.download_root = download_root
|
self.download_root = download_root
|
||||||
@@ -187,7 +189,6 @@ class PostProcessorWorker:
|
|||||||
self.multipart_scope = multipart_scope
|
self.multipart_scope = multipart_scope
|
||||||
self.multipart_parts_count = multipart_parts_count
|
self.multipart_parts_count = multipart_parts_count
|
||||||
self.multipart_min_size_mb = multipart_min_size_mb
|
self.multipart_min_size_mb = multipart_min_size_mb
|
||||||
self.skip_file_size_mb = skip_file_size_mb
|
|
||||||
if self.compress_images and Image is None:
|
if self.compress_images and Image is None:
|
||||||
self.logger("⚠️ Image compression disabled: Pillow library not found.")
|
self.logger("⚠️ Image compression disabled: Pillow library not found.")
|
||||||
self.compress_images = False
|
self.compress_images = False
|
||||||
@@ -267,35 +268,15 @@ class PostProcessorWorker:
|
|||||||
return 0, 1, "", False, FILE_DOWNLOAD_STATUS_SKIPPED, None
|
return 0, 1, "", False, FILE_DOWNLOAD_STATUS_SKIPPED, None
|
||||||
|
|
||||||
file_download_headers = {
|
file_download_headers = {
|
||||||
'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
|
||||||
'Referer': post_page_url,
|
'Referer': post_page_url
|
||||||
'Accept': 'text/css'
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
file_url = file_info.get('url')
|
file_url = file_info.get('url')
|
||||||
cookies_to_use_for_file = None
|
cookies_to_use_for_file = None
|
||||||
if self.use_cookie:
|
if self.use_cookie:
|
||||||
cookies_to_use_for_file = prepare_cookies_for_request(self.use_cookie, self.cookie_text, self.selected_cookie_file, self.app_base_dir, self.logger)
|
cookies_to_use_for_file = prepare_cookies_for_request(self.use_cookie, self.cookie_text, self.selected_cookie_file, self.app_base_dir, self.logger)
|
||||||
|
|
||||||
if self.skip_file_size_mb is not None:
|
|
||||||
api_original_filename_for_size_check = file_info.get('_original_name_for_log', file_info.get('name'))
|
|
||||||
try:
|
|
||||||
# Use a stream=True HEAD request to get headers without downloading the body
|
|
||||||
with requests.head(file_url, headers=file_download_headers, timeout=15, cookies=cookies_to_use_for_file, allow_redirects=True) as head_response:
|
|
||||||
head_response.raise_for_status()
|
|
||||||
content_length = head_response.headers.get('Content-Length')
|
|
||||||
if content_length:
|
|
||||||
file_size_bytes = int(content_length)
|
|
||||||
file_size_mb = file_size_bytes / (1024 * 1024)
|
|
||||||
if file_size_mb < self.skip_file_size_mb:
|
|
||||||
self.logger(f" -> Skip File (Size): '{api_original_filename_for_size_check}' is {file_size_mb:.2f} MB, which is smaller than the {self.skip_file_size_mb} MB limit.")
|
|
||||||
return 0, 1, api_original_filename_for_size_check, False, FILE_DOWNLOAD_STATUS_SKIPPED, None
|
|
||||||
else:
|
|
||||||
self.logger(f" ⚠️ Could not determine file size for '{api_original_filename_for_size_check}' to check against size limit. Proceeding with download.")
|
|
||||||
except requests.RequestException as e:
|
|
||||||
self.logger(f" ⚠️ Could not fetch file headers to check size for '{api_original_filename_for_size_check}': {e}. Proceeding with download.")
|
|
||||||
|
|
||||||
api_original_filename = file_info.get('_original_name_for_log', file_info.get('name'))
|
api_original_filename = file_info.get('_original_name_for_log', file_info.get('name'))
|
||||||
filename_to_save_in_main_path = ""
|
filename_to_save_in_main_path = ""
|
||||||
if forced_filename_override:
|
if forced_filename_override:
|
||||||
@@ -428,26 +409,8 @@ class PostProcessorWorker:
|
|||||||
self.logger(f"⚠️ Manga mode: Generated filename was empty. Using generic fallback: '{filename_to_save_in_main_path}'.")
|
self.logger(f"⚠️ Manga mode: Generated filename was empty. Using generic fallback: '{filename_to_save_in_main_path}'.")
|
||||||
was_original_name_kept_flag = False
|
was_original_name_kept_flag = False
|
||||||
else:
|
else:
|
||||||
is_url_like = 'http' in api_original_filename.lower()
|
filename_to_save_in_main_path = cleaned_original_api_filename
|
||||||
is_too_long = len(cleaned_original_api_filename) > 100
|
was_original_name_kept_flag = True
|
||||||
|
|
||||||
if is_url_like or is_too_long:
|
|
||||||
self.logger(f" ⚠️ Original filename is a URL or too long. Generating a shorter name.")
|
|
||||||
name_hash = hashlib.md5(api_original_filename.encode()).hexdigest()[:12]
|
|
||||||
_, ext = os.path.splitext(cleaned_original_api_filename)
|
|
||||||
if not ext:
|
|
||||||
try:
|
|
||||||
path = urlparse(api_original_filename).path
|
|
||||||
ext = os.path.splitext(path)[1] or ".file"
|
|
||||||
except Exception:
|
|
||||||
ext = ".file"
|
|
||||||
|
|
||||||
cleaned_post_title = robust_clean_name(post_title.strip() if post_title else "post")[:40]
|
|
||||||
filename_to_save_in_main_path = f"{cleaned_post_title}_{name_hash}{ext}"
|
|
||||||
was_original_name_kept_flag = False
|
|
||||||
else:
|
|
||||||
filename_to_save_in_main_path = cleaned_original_api_filename
|
|
||||||
was_original_name_kept_flag = True
|
|
||||||
|
|
||||||
if self.remove_from_filename_words_list and filename_to_save_in_main_path:
|
if self.remove_from_filename_words_list and filename_to_save_in_main_path:
|
||||||
base_name_for_removal, ext_for_removal = os.path.splitext(filename_to_save_in_main_path)
|
base_name_for_removal, ext_for_removal = os.path.splitext(filename_to_save_in_main_path)
|
||||||
@@ -525,18 +488,19 @@ class PostProcessorWorker:
|
|||||||
except requests.RequestException as e:
|
except requests.RequestException as e:
|
||||||
self.logger(f" ⚠️ Could not verify size of existing file '{filename_to_save_in_main_path}': {e}. Proceeding with download.")
|
self.logger(f" ⚠️ Could not verify size of existing file '{filename_to_save_in_main_path}': {e}. Proceeding with download.")
|
||||||
|
|
||||||
max_retries = 3
|
|
||||||
retry_delay = 5
|
retry_delay = 5
|
||||||
downloaded_size_bytes = 0
|
downloaded_size_bytes = 0
|
||||||
calculated_file_hash = None
|
calculated_file_hash = None
|
||||||
downloaded_part_file_path = None
|
downloaded_part_file_path = None
|
||||||
|
total_size_bytes = 0
|
||||||
download_successful_flag = False
|
download_successful_flag = False
|
||||||
last_exception_for_retry_later = None
|
last_exception_for_retry_later = None
|
||||||
is_permanent_error = False
|
is_permanent_error = False
|
||||||
data_to_write_io = None
|
data_to_write_io = None
|
||||||
|
|
||||||
|
response_for_this_attempt = None
|
||||||
for attempt_num_single_stream in range(max_retries + 1):
|
for attempt_num_single_stream in range(max_retries + 1):
|
||||||
response = None
|
response_for_this_attempt = None
|
||||||
if self._check_pause(f"File download attempt for '{api_original_filename}'"): break
|
if self._check_pause(f"File download attempt for '{api_original_filename}'"): break
|
||||||
if self.check_cancel() or (skip_event and skip_event.is_set()): break
|
if self.check_cancel() or (skip_event and skip_event.is_set()): break
|
||||||
try:
|
try:
|
||||||
@@ -555,24 +519,12 @@ class PostProcessorWorker:
|
|||||||
new_url = self._find_valid_subdomain(current_url_to_try)
|
new_url = self._find_valid_subdomain(current_url_to_try)
|
||||||
if new_url != current_url_to_try:
|
if new_url != current_url_to_try:
|
||||||
self.logger(f" Retrying with new URL: {new_url}")
|
self.logger(f" Retrying with new URL: {new_url}")
|
||||||
file_url = new_url
|
file_url = new_url # Update the main file_url for subsequent retries
|
||||||
response.close() # Close the old response
|
|
||||||
response = requests.get(new_url, headers=file_download_headers, timeout=(30, 300), stream=True, cookies=cookies_to_use_for_file)
|
response = requests.get(new_url, headers=file_download_headers, timeout=(30, 300), stream=True, cookies=cookies_to_use_for_file)
|
||||||
|
|
||||||
|
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
|
|
||||||
# --- REVISED AND MOVED SIZE CHECK LOGIC ---
|
|
||||||
total_size_bytes = int(response.headers.get('Content-Length', 0))
|
total_size_bytes = int(response.headers.get('Content-Length', 0))
|
||||||
|
|
||||||
if self.skip_file_size_mb is not None:
|
|
||||||
if total_size_bytes > 0:
|
|
||||||
file_size_mb = total_size_bytes / (1024 * 1024)
|
|
||||||
if file_size_mb < self.skip_file_size_mb:
|
|
||||||
self.logger(f" -> Skip File (Size): '{api_original_filename}' is {file_size_mb:.2f} MB, which is smaller than the {self.skip_file_size_mb} MB limit.")
|
|
||||||
return 0, 1, api_original_filename, False, FILE_DOWNLOAD_STATUS_SKIPPED, None
|
|
||||||
# If Content-Length is missing, we can't check, so we no longer log a warning here and just proceed.
|
|
||||||
# --- END OF REVISED LOGIC ---
|
|
||||||
|
|
||||||
num_parts_for_file = min(self.multipart_parts_count, MAX_PARTS_FOR_MULTIPART_DOWNLOAD)
|
num_parts_for_file = min(self.multipart_parts_count, MAX_PARTS_FOR_MULTIPART_DOWNLOAD)
|
||||||
|
|
||||||
file_is_eligible_by_scope = False
|
file_is_eligible_by_scope = False
|
||||||
@@ -596,7 +548,9 @@ class PostProcessorWorker:
|
|||||||
if self._check_pause(f"Multipart decision for '{api_original_filename}'"): break
|
if self._check_pause(f"Multipart decision for '{api_original_filename}'"): break
|
||||||
|
|
||||||
if attempt_multipart:
|
if attempt_multipart:
|
||||||
response.close() # Close the initial connection before starting multipart
|
if response_for_this_attempt:
|
||||||
|
response_for_this_attempt.close()
|
||||||
|
response_for_this_attempt = None
|
||||||
mp_save_path_for_unique_part_stem_arg = os.path.join(target_folder_path, f"{unique_part_file_stem_on_disk}{temp_file_ext_for_unique_part}")
|
mp_save_path_for_unique_part_stem_arg = os.path.join(target_folder_path, f"{unique_part_file_stem_on_disk}{temp_file_ext_for_unique_part}")
|
||||||
mp_success, mp_bytes, mp_hash, mp_file_handle = download_file_in_parts(
|
mp_success, mp_bytes, mp_hash, mp_file_handle = download_file_in_parts(
|
||||||
file_url, mp_save_path_for_unique_part_stem_arg, total_size_bytes, num_parts_for_file, file_download_headers, api_original_filename,
|
file_url, mp_save_path_for_unique_part_stem_arg, total_size_bytes, num_parts_for_file, file_download_headers, api_original_filename,
|
||||||
@@ -622,6 +576,7 @@ class PostProcessorWorker:
|
|||||||
current_attempt_downloaded_bytes = 0
|
current_attempt_downloaded_bytes = 0
|
||||||
md5_hasher = hashlib.md5()
|
md5_hasher = hashlib.md5()
|
||||||
last_progress_time = time.time()
|
last_progress_time = time.time()
|
||||||
|
single_stream_exception = None
|
||||||
try:
|
try:
|
||||||
with open(current_single_stream_part_path, 'wb') as f_part:
|
with open(current_single_stream_part_path, 'wb') as f_part:
|
||||||
for chunk in response.iter_content(chunk_size=1 * 1024 * 1024):
|
for chunk in response.iter_content(chunk_size=1 * 1024 * 1024):
|
||||||
@@ -688,8 +643,8 @@ class PostProcessorWorker:
|
|||||||
is_permanent_error = True
|
is_permanent_error = True
|
||||||
break
|
break
|
||||||
finally:
|
finally:
|
||||||
if response:
|
if response_for_this_attempt:
|
||||||
response.close()
|
response_for_this_attempt.close()
|
||||||
self._emit_signal('file_download_status', False)
|
self._emit_signal('file_download_status', False)
|
||||||
|
|
||||||
final_total_for_progress = total_size_bytes if download_successful_flag and total_size_bytes > 0 else downloaded_size_bytes
|
final_total_for_progress = total_size_bytes if download_successful_flag and total_size_bytes > 0 else downloaded_size_bytes
|
||||||
@@ -871,7 +826,9 @@ class PostProcessorWorker:
|
|||||||
return 0, 1, filename_to_save_in_main_path, was_original_name_kept_flag, FILE_DOWNLOAD_STATUS_FAILED_RETRYABLE_LATER, details_for_failure
|
return 0, 1, filename_to_save_in_main_path, was_original_name_kept_flag, FILE_DOWNLOAD_STATUS_FAILED_RETRYABLE_LATER, details_for_failure
|
||||||
|
|
||||||
def process(self):
|
def process(self):
|
||||||
|
# --- START: REFACTORED PROCESS METHOD ---
|
||||||
|
|
||||||
|
# 1. DATA MAPPING: Map Discord Message or Creator Post fields to a consistent set of variables.
|
||||||
if self.service == 'discord':
|
if self.service == 'discord':
|
||||||
# For Discord, self.post is a MESSAGE object from the API.
|
# For Discord, self.post is a MESSAGE object from the API.
|
||||||
post_title = self.post.get('content', '') or f"Message {self.post.get('id', 'N/A')}"
|
post_title = self.post.get('content', '') or f"Message {self.post.get('id', 'N/A')}"
|
||||||
@@ -891,43 +848,7 @@ class PostProcessorWorker:
|
|||||||
post_data = self.post # Reference to the post object
|
post_data = self.post # Reference to the post object
|
||||||
log_prefix = "Post"
|
log_prefix = "Post"
|
||||||
|
|
||||||
# --- FIX: FETCH FULL POST DATA IF CONTENT IS MISSING BUT NEEDED ---
|
# 2. SHARED PROCESSING LOGIC: The rest of the function now uses the consistent variables from above.
|
||||||
content_is_needed = (
|
|
||||||
self.show_external_links or
|
|
||||||
self.extract_links_only or
|
|
||||||
self.scan_content_for_images or
|
|
||||||
(self.filter_mode == 'text_only' and self.text_only_scope == 'content')
|
|
||||||
)
|
|
||||||
|
|
||||||
if content_is_needed and self.post.get('content') is None and self.service != 'discord':
|
|
||||||
|
|
||||||
self.logger(f" Post {post_id} is missing 'content' field, fetching full data...")
|
|
||||||
parsed_url = urlparse(self.api_url_input)
|
|
||||||
api_domain = parsed_url.netloc
|
|
||||||
creator_page_url = f"https://{api_domain}/{self.service}/user/{self.user_id}"
|
|
||||||
headers = {
|
|
||||||
'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
|
|
||||||
'Referer': creator_page_url,
|
|
||||||
'Accept': 'text/css'
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
cookies = prepare_cookies_for_request(self.use_cookie, self.cookie_text, self.selected_cookie_file, self.app_base_dir, self.logger, target_domain=api_domain)
|
|
||||||
|
|
||||||
full_post_data = fetch_single_post_data(api_domain, self.service, self.user_id, post_id, headers, self.logger, cookies_dict=cookies)
|
|
||||||
|
|
||||||
if full_post_data:
|
|
||||||
self.logger(" ✅ Full post data fetched successfully.")
|
|
||||||
self.post = full_post_data
|
|
||||||
post_title = self.post.get('title', '') or 'untitled_post'
|
|
||||||
post_main_file_info = self.post.get('file')
|
|
||||||
post_attachments = self.post.get('attachments', [])
|
|
||||||
post_content_html = self.post.get('content', '')
|
|
||||||
post_data = self.post
|
|
||||||
else:
|
|
||||||
self.logger(f" ⚠️ Failed to fetch full content for post {post_id}. Content-dependent features may not work for this post.")
|
|
||||||
|
|
||||||
result_tuple = (0, 0, [], [], [], None, None)
|
result_tuple = (0, 0, [], [], [], None, None)
|
||||||
total_downloaded_this_post = 0
|
total_downloaded_this_post = 0
|
||||||
total_skipped_this_post = 0
|
total_skipped_this_post = 0
|
||||||
@@ -956,11 +877,7 @@ class PostProcessorWorker:
|
|||||||
else:
|
else:
|
||||||
post_page_url = f"https://{parsed_api_url.netloc}/{self.service}/user/{self.user_id}/post/{post_id}"
|
post_page_url = f"https://{parsed_api_url.netloc}/{self.service}/user/{self.user_id}/post/{post_id}"
|
||||||
|
|
||||||
headers = {
|
headers = {'User-Agent': 'Mozilla/5.0', 'Referer': post_page_url, 'Accept': '*/*'}
|
||||||
'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
|
|
||||||
'Referer': post_page_url,
|
|
||||||
'Accept': 'text/css'
|
|
||||||
}
|
|
||||||
link_pattern = re.compile(r"""<a\s+.*?href=["'](https?://[^"']+)["'][^>]*>(.*?)</a>""", re.IGNORECASE | re.DOTALL)
|
link_pattern = re.compile(r"""<a\s+.*?href=["'](https?://[^"']+)["'][^>]*>(.*?)</a>""", re.IGNORECASE | re.DOTALL)
|
||||||
|
|
||||||
effective_unwanted_keywords_for_folder_naming = self.unwanted_keywords.copy()
|
effective_unwanted_keywords_for_folder_naming = self.unwanted_keywords.copy()
|
||||||
@@ -1341,6 +1258,7 @@ class PostProcessorWorker:
|
|||||||
parsed_url = urlparse(self.api_url_input)
|
parsed_url = urlparse(self.api_url_input)
|
||||||
api_domain = parsed_url.netloc
|
api_domain = parsed_url.netloc
|
||||||
cookies = prepare_cookies_for_request(self.use_cookie, self.cookie_text, self.selected_cookie_file, self.app_base_dir, self.logger, target_domain=api_domain)
|
cookies = prepare_cookies_for_request(self.use_cookie, self.cookie_text, self.selected_cookie_file, self.app_base_dir, self.logger, target_domain=api_domain)
|
||||||
|
from .api_client import fetch_single_post_data
|
||||||
full_data = fetch_single_post_data(api_domain, self.service, self.user_id, post_id, headers, self.logger, cookies_dict=cookies)
|
full_data = fetch_single_post_data(api_domain, self.service, self.user_id, post_id, headers, self.logger, cookies_dict=cookies)
|
||||||
if full_data:
|
if full_data:
|
||||||
final_post_data = full_data
|
final_post_data = full_data
|
||||||
@@ -2017,9 +1935,7 @@ class DownloadThread(QThread):
|
|||||||
project_root_dir=None,
|
project_root_dir=None,
|
||||||
processed_post_ids=None,
|
processed_post_ids=None,
|
||||||
start_offset=0,
|
start_offset=0,
|
||||||
fetch_first=False,
|
fetch_first=False):
|
||||||
skip_file_size_mb=None
|
|
||||||
):
|
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.api_url_input = api_url_input
|
self.api_url_input = api_url_input
|
||||||
self.output_dir = output_dir
|
self.output_dir = output_dir
|
||||||
@@ -2086,7 +2002,6 @@ class DownloadThread(QThread):
|
|||||||
self.processed_post_ids_set = set(processed_post_ids) if processed_post_ids is not None else set()
|
self.processed_post_ids_set = set(processed_post_ids) if processed_post_ids is not None else set()
|
||||||
self.start_offset = start_offset
|
self.start_offset = start_offset
|
||||||
self.fetch_first = fetch_first
|
self.fetch_first = fetch_first
|
||||||
self.skip_file_size_mb = skip_file_size_mb
|
|
||||||
|
|
||||||
if self.compress_images and Image is None:
|
if self.compress_images and Image is None:
|
||||||
self.logger("⚠️ Image compression disabled: Pillow library not found (DownloadThread).")
|
self.logger("⚠️ Image compression disabled: Pillow library not found (DownloadThread).")
|
||||||
@@ -2207,7 +2122,6 @@ class DownloadThread(QThread):
|
|||||||
'single_pdf_mode': self.single_pdf_mode,
|
'single_pdf_mode': self.single_pdf_mode,
|
||||||
'multipart_parts_count': self.multipart_parts_count,
|
'multipart_parts_count': self.multipart_parts_count,
|
||||||
'multipart_min_size_mb': self.multipart_min_size_mb,
|
'multipart_min_size_mb': self.multipart_min_size_mb,
|
||||||
'skip_file_size_mb': self.skip_file_size_mb,
|
|
||||||
'project_root_dir': self.project_root_dir,
|
'project_root_dir': self.project_root_dir,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -5,12 +5,9 @@ import traceback
|
|||||||
import json
|
import json
|
||||||
import base64
|
import base64
|
||||||
import time
|
import time
|
||||||
import zipfile
|
|
||||||
from urllib.parse import urlparse, urlunparse, parse_qs, urlencode
|
from urllib.parse import urlparse, urlunparse, parse_qs, urlencode
|
||||||
|
|
||||||
# --- Third-party Library Imports ---
|
|
||||||
import requests
|
import requests
|
||||||
import cloudscraper
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from Crypto.Cipher import AES
|
from Crypto.Cipher import AES
|
||||||
@@ -29,12 +26,12 @@ MEGA_API_URL = "https://g.api.mega.co.nz"
|
|||||||
def _get_filename_from_headers(headers):
|
def _get_filename_from_headers(headers):
|
||||||
"""
|
"""
|
||||||
Extracts a filename from the Content-Disposition header.
|
Extracts a filename from the Content-Disposition header.
|
||||||
|
(This is from your original file and is kept for Dropbox downloads)
|
||||||
"""
|
"""
|
||||||
cd = headers.get('content-disposition')
|
cd = headers.get('content-disposition')
|
||||||
if not cd:
|
if not cd:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# Handles both filename="file.zip" and filename*=UTF-8''file%20name.zip
|
|
||||||
fname_match = re.findall('filename="?([^"]+)"?', cd)
|
fname_match = re.findall('filename="?([^"]+)"?', cd)
|
||||||
if fname_match:
|
if fname_match:
|
||||||
sanitized_name = re.sub(r'[<>:"/\\|?*]', '_', fname_match[0].strip())
|
sanitized_name = re.sub(r'[<>:"/\\|?*]', '_', fname_match[0].strip())
|
||||||
@@ -42,23 +39,28 @@ def _get_filename_from_headers(headers):
|
|||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# --- Helper functions for Mega decryption ---
|
# --- NEW: Helper functions for Mega decryption ---
|
||||||
|
|
||||||
def urlb64_to_b64(s):
|
def urlb64_to_b64(s):
|
||||||
|
"""Converts a URL-safe base64 string to a standard base64 string."""
|
||||||
s = s.replace('-', '+').replace('_', '/')
|
s = s.replace('-', '+').replace('_', '/')
|
||||||
s += '=' * (-len(s) % 4)
|
s += '=' * (-len(s) % 4)
|
||||||
return s
|
return s
|
||||||
|
|
||||||
def b64_to_bytes(s):
|
def b64_to_bytes(s):
|
||||||
|
"""Decodes a URL-safe base64 string to bytes."""
|
||||||
return base64.b64decode(urlb64_to_b64(s))
|
return base64.b64decode(urlb64_to_b64(s))
|
||||||
|
|
||||||
def bytes_to_hex(b):
|
def bytes_to_hex(b):
|
||||||
|
"""Converts bytes to a hex string."""
|
||||||
return b.hex()
|
return b.hex()
|
||||||
|
|
||||||
def hex_to_bytes(h):
|
def hex_to_bytes(h):
|
||||||
|
"""Converts a hex string to bytes."""
|
||||||
return bytes.fromhex(h)
|
return bytes.fromhex(h)
|
||||||
|
|
||||||
def hrk2hk(hex_raw_key):
|
def hrk2hk(hex_raw_key):
|
||||||
|
"""Derives the final AES key from the raw key components for Mega."""
|
||||||
key_part1 = int(hex_raw_key[0:16], 16)
|
key_part1 = int(hex_raw_key[0:16], 16)
|
||||||
key_part2 = int(hex_raw_key[16:32], 16)
|
key_part2 = int(hex_raw_key[16:32], 16)
|
||||||
key_part3 = int(hex_raw_key[32:48], 16)
|
key_part3 = int(hex_raw_key[32:48], 16)
|
||||||
@@ -70,20 +72,23 @@ def hrk2hk(hex_raw_key):
|
|||||||
return f'{final_key_part1:016x}{final_key_part2:016x}'
|
return f'{final_key_part1:016x}{final_key_part2:016x}'
|
||||||
|
|
||||||
def decrypt_at(at_b64, key_bytes):
|
def decrypt_at(at_b64, key_bytes):
|
||||||
|
"""Decrypts the 'at' attribute to get file metadata."""
|
||||||
at_bytes = b64_to_bytes(at_b64)
|
at_bytes = b64_to_bytes(at_b64)
|
||||||
iv = b'\0' * 16
|
iv = b'\0' * 16
|
||||||
cipher = AES.new(key_bytes, AES.MODE_CBC, iv)
|
cipher = AES.new(key_bytes, AES.MODE_CBC, iv)
|
||||||
decrypted_at = cipher.decrypt(at_bytes)
|
decrypted_at = cipher.decrypt(at_bytes)
|
||||||
return decrypted_at.decode('utf-8').strip('\0').replace('MEGA', '')
|
return decrypted_at.decode('utf-8').strip('\0').replace('MEGA', '')
|
||||||
|
|
||||||
# --- Core Logic for Mega Downloads ---
|
# --- NEW: Core Logic for Mega Downloads ---
|
||||||
|
|
||||||
def get_mega_file_info(file_id, file_key, session, logger_func):
|
def get_mega_file_info(file_id, file_key, session, logger_func):
|
||||||
|
"""Fetches file metadata and the temporary download URL from the Mega API."""
|
||||||
try:
|
try:
|
||||||
hex_raw_key = bytes_to_hex(b64_to_bytes(file_key))
|
hex_raw_key = bytes_to_hex(b64_to_bytes(file_key))
|
||||||
hex_key = hrk2hk(hex_raw_key)
|
hex_key = hrk2hk(hex_raw_key)
|
||||||
key_bytes = hex_to_bytes(hex_key)
|
key_bytes = hex_to_bytes(hex_key)
|
||||||
|
|
||||||
|
# Request file attributes
|
||||||
payload = [{"a": "g", "p": file_id}]
|
payload = [{"a": "g", "p": file_id}]
|
||||||
response = session.post(f"{MEGA_API_URL}/cs", json=payload, timeout=20)
|
response = session.post(f"{MEGA_API_URL}/cs", json=payload, timeout=20)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
@@ -95,10 +100,13 @@ def get_mega_file_info(file_id, file_key, session, logger_func):
|
|||||||
|
|
||||||
file_size = res_json[0]['s']
|
file_size = res_json[0]['s']
|
||||||
at_b64 = res_json[0]['at']
|
at_b64 = res_json[0]['at']
|
||||||
|
|
||||||
|
# Decrypt attributes to get the file name
|
||||||
at_dec_json_str = decrypt_at(at_b64, key_bytes)
|
at_dec_json_str = decrypt_at(at_b64, key_bytes)
|
||||||
at_dec_json = json.loads(at_dec_json_str)
|
at_dec_json = json.loads(at_dec_json_str)
|
||||||
file_name = at_dec_json['n']
|
file_name = at_dec_json['n']
|
||||||
|
|
||||||
|
# Request the temporary download URL
|
||||||
payload = [{"a": "g", "g": 1, "p": file_id}]
|
payload = [{"a": "g", "g": 1, "p": file_id}]
|
||||||
response = session.post(f"{MEGA_API_URL}/cs", json=payload, timeout=20)
|
response = session.post(f"{MEGA_API_URL}/cs", json=payload, timeout=20)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
@@ -116,16 +124,19 @@ def get_mega_file_info(file_id, file_key, session, logger_func):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
def download_and_decrypt_mega_file(info, download_path, logger_func):
|
def download_and_decrypt_mega_file(info, download_path, logger_func):
|
||||||
|
"""Downloads the file and decrypts it chunk by chunk, reporting progress."""
|
||||||
file_name = info['file_name']
|
file_name = info['file_name']
|
||||||
file_size = info['file_size']
|
file_size = info['file_size']
|
||||||
dl_url = info['dl_url']
|
dl_url = info['dl_url']
|
||||||
hex_raw_key = info['hex_raw_key']
|
hex_raw_key = info['hex_raw_key']
|
||||||
|
|
||||||
final_path = os.path.join(download_path, file_name)
|
final_path = os.path.join(download_path, file_name)
|
||||||
|
|
||||||
if os.path.exists(final_path) and os.path.getsize(final_path) == file_size:
|
if os.path.exists(final_path) and os.path.getsize(final_path) == file_size:
|
||||||
logger_func(f" [Mega] ℹ️ File '{file_name}' already exists with the correct size. Skipping.")
|
logger_func(f" [Mega] ℹ️ File '{file_name}' already exists with the correct size. Skipping.")
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# Prepare for decryption
|
||||||
key = hex_to_bytes(hrk2hk(hex_raw_key))
|
key = hex_to_bytes(hrk2hk(hex_raw_key))
|
||||||
iv_hex = hex_raw_key[32:48] + '0000000000000000'
|
iv_hex = hex_raw_key[32:48] + '0000000000000000'
|
||||||
iv_bytes = hex_to_bytes(iv_hex)
|
iv_bytes = hex_to_bytes(iv_hex)
|
||||||
@@ -139,11 +150,13 @@ def download_and_decrypt_mega_file(info, download_path, logger_func):
|
|||||||
|
|
||||||
with open(final_path, 'wb') as f:
|
with open(final_path, 'wb') as f:
|
||||||
for chunk in r.iter_content(chunk_size=8192):
|
for chunk in r.iter_content(chunk_size=8192):
|
||||||
if not chunk: continue
|
if not chunk:
|
||||||
|
continue
|
||||||
decrypted_chunk = cipher.decrypt(chunk)
|
decrypted_chunk = cipher.decrypt(chunk)
|
||||||
f.write(decrypted_chunk)
|
f.write(decrypted_chunk)
|
||||||
downloaded_bytes += len(chunk)
|
downloaded_bytes += len(chunk)
|
||||||
|
|
||||||
|
# Log progress every second
|
||||||
current_time = time.time()
|
current_time = time.time()
|
||||||
if current_time - last_log_time > 1:
|
if current_time - last_log_time > 1:
|
||||||
progress_percent = (downloaded_bytes / file_size) * 100 if file_size > 0 else 0
|
progress_percent = (downloaded_bytes / file_size) * 100 if file_size > 0 else 0
|
||||||
@@ -151,16 +164,28 @@ def download_and_decrypt_mega_file(info, download_path, logger_func):
|
|||||||
last_log_time = current_time
|
last_log_time = current_time
|
||||||
|
|
||||||
logger_func(f" [Mega] ✅ Successfully downloaded '{file_name}' to '{download_path}'")
|
logger_func(f" [Mega] ✅ Successfully downloaded '{file_name}' to '{download_path}'")
|
||||||
|
except requests.RequestException as e:
|
||||||
|
logger_func(f" [Mega] ❌ Download failed for '{file_name}': {e}")
|
||||||
|
except IOError as e:
|
||||||
|
logger_func(f" [Mega] ❌ Could not write to file '{final_path}': {e}")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger_func(f" [Mega] ❌ An unexpected error occurred during download/decryption: {e}")
|
logger_func(f" [Mega] ❌ An unexpected error occurred during download/decryption: {e}")
|
||||||
|
|
||||||
|
|
||||||
|
# --- REPLACEMENT Main Service Downloader Function for Mega ---
|
||||||
|
|
||||||
def download_mega_file(mega_url, download_path, logger_func=print):
|
def download_mega_file(mega_url, download_path, logger_func=print):
|
||||||
|
"""
|
||||||
|
Downloads a file from a Mega.nz URL using direct requests and decryption.
|
||||||
|
This replaces the old mega.py implementation.
|
||||||
|
"""
|
||||||
if not PYCRYPTODOME_AVAILABLE:
|
if not PYCRYPTODOME_AVAILABLE:
|
||||||
logger_func("❌ Mega download failed: 'pycryptodome' library is not installed. Please run: pip install pycryptodome")
|
logger_func("❌ Mega download failed: 'pycryptodome' library is not installed. Please run: pip install pycryptodome")
|
||||||
return
|
return
|
||||||
|
|
||||||
logger_func(f" [Mega] Initializing download for: {mega_url}")
|
logger_func(f" [Mega] Initializing download for: {mega_url}")
|
||||||
|
|
||||||
|
# Regex to capture file ID and key from both old and new URL formats
|
||||||
match = re.search(r'mega(?:\.co)?\.nz/(?:file/|#!)?([a-zA-Z0-9]+)(?:#|!)([a-zA-Z0-9_.-]+)', mega_url)
|
match = re.search(r'mega(?:\.co)?\.nz/(?:file/|#!)?([a-zA-Z0-9]+)(?:#|!)([a-zA-Z0-9_.-]+)', mega_url)
|
||||||
if not match:
|
if not match:
|
||||||
logger_func(f" [Mega] ❌ Error: Invalid Mega URL format.")
|
logger_func(f" [Mega] ❌ Error: Invalid Mega URL format.")
|
||||||
@@ -174,14 +199,18 @@ def download_mega_file(mega_url, download_path, logger_func=print):
|
|||||||
|
|
||||||
file_info = get_mega_file_info(file_id, file_key, session, logger_func)
|
file_info = get_mega_file_info(file_id, file_key, session, logger_func)
|
||||||
if not file_info:
|
if not file_info:
|
||||||
logger_func(f" [Mega] ❌ Failed to get file info. Aborting.")
|
logger_func(f" [Mega] ❌ Failed to get file info. The link may be invalid or expired. Aborting.")
|
||||||
return
|
return
|
||||||
|
|
||||||
logger_func(f" [Mega] File found: '{file_info['file_name']}' (Size: {file_info['file_size'] / 1024 / 1024:.2f} MB)")
|
logger_func(f" [Mega] File found: '{file_info['file_name']}' (Size: {file_info['file_size'] / 1024 / 1024:.2f} MB)")
|
||||||
|
|
||||||
download_and_decrypt_mega_file(file_info, download_path, logger_func)
|
download_and_decrypt_mega_file(file_info, download_path, logger_func)
|
||||||
|
|
||||||
|
|
||||||
|
# --- ORIGINAL Functions for Google Drive and Dropbox (Unchanged) ---
|
||||||
|
|
||||||
def download_gdrive_file(url, download_path, logger_func=print):
|
def download_gdrive_file(url, download_path, logger_func=print):
|
||||||
|
"""Downloads a file from a Google Drive link."""
|
||||||
if not GDRIVE_AVAILABLE:
|
if not GDRIVE_AVAILABLE:
|
||||||
logger_func("❌ Google Drive download failed: 'gdown' library is not installed.")
|
logger_func("❌ Google Drive download failed: 'gdown' library is not installed.")
|
||||||
return
|
return
|
||||||
@@ -198,15 +227,12 @@ def download_gdrive_file(url, download_path, logger_func=print):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger_func(f" [G-Drive] ❌ An unexpected error occurred: {e}")
|
logger_func(f" [G-Drive] ❌ An unexpected error occurred: {e}")
|
||||||
|
|
||||||
# --- MODIFIED DROPBOX DOWNLOADER ---
|
|
||||||
def download_dropbox_file(dropbox_link, download_path=".", logger_func=print):
|
def download_dropbox_file(dropbox_link, download_path=".", logger_func=print):
|
||||||
"""
|
"""
|
||||||
Downloads a file or a folder (as a zip) from a public Dropbox link.
|
Downloads a file from a public Dropbox link by modifying the URL for direct download.
|
||||||
Uses cloudscraper to handle potential browser checks and auto-extracts zip files.
|
|
||||||
"""
|
"""
|
||||||
logger_func(f" [Dropbox] Attempting to download: {dropbox_link}")
|
logger_func(f" [Dropbox] Attempting to download: {dropbox_link}")
|
||||||
|
|
||||||
# Modify URL to force download (works for both files and folders)
|
|
||||||
parsed_url = urlparse(dropbox_link)
|
parsed_url = urlparse(dropbox_link)
|
||||||
query_params = parse_qs(parsed_url.query)
|
query_params = parse_qs(parsed_url.query)
|
||||||
query_params['dl'] = ['1']
|
query_params['dl'] = ['1']
|
||||||
@@ -215,60 +241,26 @@ def download_dropbox_file(dropbox_link, download_path=".", logger_func=print):
|
|||||||
|
|
||||||
logger_func(f" [Dropbox] Using direct download URL: {direct_download_url}")
|
logger_func(f" [Dropbox] Using direct download URL: {direct_download_url}")
|
||||||
|
|
||||||
scraper = cloudscraper.create_scraper()
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if not os.path.exists(download_path):
|
if not os.path.exists(download_path):
|
||||||
os.makedirs(download_path, exist_ok=True)
|
os.makedirs(download_path, exist_ok=True)
|
||||||
logger_func(f" [Dropbox] Created download directory: {download_path}")
|
logger_func(f" [Dropbox] Created download directory: {download_path}")
|
||||||
|
|
||||||
with scraper.get(direct_download_url, stream=True, allow_redirects=True, timeout=(20, 600)) as r:
|
with requests.get(direct_download_url, stream=True, allow_redirects=True, timeout=(10, 300)) as r:
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
|
|
||||||
filename = _get_filename_from_headers(r.headers) or os.path.basename(parsed_url.path) or "dropbox_download"
|
filename = _get_filename_from_headers(r.headers) or os.path.basename(parsed_url.path) or "dropbox_file"
|
||||||
# If it's a folder, Dropbox will name it FolderName.zip
|
|
||||||
if not os.path.splitext(filename)[1]:
|
|
||||||
filename += ".zip"
|
|
||||||
|
|
||||||
full_save_path = os.path.join(download_path, filename)
|
full_save_path = os.path.join(download_path, filename)
|
||||||
|
|
||||||
logger_func(f" [Dropbox] Starting download of '{filename}'...")
|
logger_func(f" [Dropbox] Starting download of '{filename}'...")
|
||||||
|
|
||||||
total_size = int(r.headers.get('content-length', 0))
|
|
||||||
downloaded_bytes = 0
|
|
||||||
last_log_time = time.time()
|
|
||||||
|
|
||||||
with open(full_save_path, 'wb') as f:
|
with open(full_save_path, 'wb') as f:
|
||||||
for chunk in r.iter_content(chunk_size=8192):
|
for chunk in r.iter_content(chunk_size=8192):
|
||||||
f.write(chunk)
|
f.write(chunk)
|
||||||
downloaded_bytes += len(chunk)
|
|
||||||
current_time = time.time()
|
|
||||||
if total_size > 0 and current_time - last_log_time > 1:
|
|
||||||
progress = (downloaded_bytes / total_size) * 100
|
|
||||||
logger_func(f" -> Downloading '{filename}'... {downloaded_bytes/1024/1024:.2f}MB / {total_size/1024/1024:.2f}MB ({progress:.1f}%)")
|
|
||||||
last_log_time = current_time
|
|
||||||
|
|
||||||
logger_func(f" [Dropbox] ✅ Download complete: {full_save_path}")
|
logger_func(f" [Dropbox] ✅ Dropbox file downloaded successfully: {full_save_path}")
|
||||||
|
|
||||||
# --- NEW: Auto-extraction logic ---
|
|
||||||
if zipfile.is_zipfile(full_save_path):
|
|
||||||
logger_func(f" [Dropbox] ዚ Detected zip file. Attempting to extract...")
|
|
||||||
extract_folder_name = os.path.splitext(filename)[0]
|
|
||||||
extract_path = os.path.join(download_path, extract_folder_name)
|
|
||||||
os.makedirs(extract_path, exist_ok=True)
|
|
||||||
|
|
||||||
with zipfile.ZipFile(full_save_path, 'r') as zip_ref:
|
|
||||||
zip_ref.extractall(extract_path)
|
|
||||||
|
|
||||||
logger_func(f" [Dropbox] ✅ Successfully extracted to folder: '{extract_path}'")
|
|
||||||
|
|
||||||
# Optional: remove the zip file after extraction
|
|
||||||
try:
|
|
||||||
os.remove(full_save_path)
|
|
||||||
logger_func(f" [Dropbox] 🗑️ Removed original zip file.")
|
|
||||||
except OSError as e:
|
|
||||||
logger_func(f" [Dropbox] ⚠️ Could not remove original zip file: {e}")
|
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger_func(f" [Dropbox] ❌ An error occurred during Dropbox download: {e}")
|
logger_func(f" [Dropbox] ❌ An error occurred during Dropbox download: {e}")
|
||||||
traceback.print_exc(limit=2)
|
traceback.print_exc(limit=2)
|
||||||
|
raise
|
||||||
|
|||||||
@@ -1,125 +0,0 @@
|
|||||||
import sys
|
|
||||||
import os
|
|
||||||
import requests
|
|
||||||
import subprocess # Keep this for now, though it's not used in the final command
|
|
||||||
from packaging.version import parse as parse_version
|
|
||||||
from PyQt5.QtCore import QThread, pyqtSignal
|
|
||||||
|
|
||||||
# Constants for the updater
|
|
||||||
GITHUB_REPO_URL = "https://api.github.com/repos/Yuvi9587/Kemono-Downloader/releases/latest"
|
|
||||||
EXE_NAME = "Kemono.Downloader.exe"
|
|
||||||
|
|
||||||
class UpdateChecker(QThread):
|
|
||||||
"""Checks for a new version on GitHub in a background thread."""
|
|
||||||
update_available = pyqtSignal(str, str) # new_version, download_url
|
|
||||||
up_to_date = pyqtSignal(str)
|
|
||||||
update_error = pyqtSignal(str)
|
|
||||||
|
|
||||||
def __init__(self, current_version):
|
|
||||||
super().__init__()
|
|
||||||
self.current_version_str = current_version.lstrip('v')
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
try:
|
|
||||||
response = requests.get(GITHUB_REPO_URL, timeout=15)
|
|
||||||
response.raise_for_status()
|
|
||||||
data = response.json()
|
|
||||||
|
|
||||||
latest_version_str = data['tag_name'].lstrip('v')
|
|
||||||
current_version = parse_version(self.current_version_str)
|
|
||||||
latest_version = parse_version(latest_version_str)
|
|
||||||
|
|
||||||
if latest_version > current_version:
|
|
||||||
for asset in data.get('assets', []):
|
|
||||||
if asset['name'] == EXE_NAME:
|
|
||||||
self.update_available.emit(latest_version_str, asset['browser_download_url'])
|
|
||||||
return
|
|
||||||
self.update_error.emit(f"Update found, but '{EXE_NAME}' is missing from the release assets.")
|
|
||||||
else:
|
|
||||||
self.up_to_date.emit("You are on the latest version.")
|
|
||||||
|
|
||||||
except requests.exceptions.RequestException as e:
|
|
||||||
self.update_error.emit(f"Network error: {e}")
|
|
||||||
except Exception as e:
|
|
||||||
self.update_error.emit(f"An error occurred: {e}")
|
|
||||||
|
|
||||||
|
|
||||||
class UpdateDownloader(QThread):
|
|
||||||
"""
|
|
||||||
Downloads the new executable and runs an updater script that kills the old process,
|
|
||||||
replaces the file, and displays a message in the terminal.
|
|
||||||
"""
|
|
||||||
download_finished = pyqtSignal()
|
|
||||||
download_error = pyqtSignal(str)
|
|
||||||
|
|
||||||
def __init__(self, download_url, parent_app):
|
|
||||||
super().__init__()
|
|
||||||
self.download_url = download_url
|
|
||||||
self.parent_app = parent_app
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
try:
|
|
||||||
app_path = sys.executable
|
|
||||||
app_dir = os.path.dirname(app_path)
|
|
||||||
temp_path = os.path.join(app_dir, f"{EXE_NAME}.tmp")
|
|
||||||
old_path = os.path.join(app_dir, f"{EXE_NAME}.old")
|
|
||||||
updater_script_path = os.path.join(app_dir, "updater.bat")
|
|
||||||
|
|
||||||
# --- NEW: Path for the PID file ---
|
|
||||||
pid_file_path = os.path.join(app_dir, "updater.pid")
|
|
||||||
|
|
||||||
# Download the new executable
|
|
||||||
with requests.get(self.download_url, stream=True, timeout=300) as r:
|
|
||||||
r.raise_for_status()
|
|
||||||
with open(temp_path, 'wb') as f:
|
|
||||||
for chunk in r.iter_content(chunk_size=8192):
|
|
||||||
f.write(chunk)
|
|
||||||
|
|
||||||
# --- NEW: Write the current Process ID to the pid file ---
|
|
||||||
with open(pid_file_path, "w") as f:
|
|
||||||
f.write(str(os.getpid()))
|
|
||||||
|
|
||||||
# --- NEW BATCH SCRIPT ---
|
|
||||||
# This script now reads the PID from the "updater.pid" file.
|
|
||||||
script_content = f"""
|
|
||||||
@echo off
|
|
||||||
SETLOCAL
|
|
||||||
|
|
||||||
echo.
|
|
||||||
echo Reading process information...
|
|
||||||
set /p PID=<{pid_file_path}
|
|
||||||
|
|
||||||
echo Closing the old application (PID: %PID%)...
|
|
||||||
taskkill /F /PID %PID%
|
|
||||||
|
|
||||||
echo Waiting for files to unlock...
|
|
||||||
timeout /t 2 /nobreak > nul
|
|
||||||
|
|
||||||
echo Replacing application files...
|
|
||||||
if exist "{old_path}" del /F /Q "{old_path}"
|
|
||||||
rename "{app_path}" "{os.path.basename(old_path)}"
|
|
||||||
rename "{temp_path}" "{EXE_NAME}"
|
|
||||||
|
|
||||||
echo.
|
|
||||||
echo ============================================================
|
|
||||||
echo Update Complete!
|
|
||||||
echo You can now close this window and run {EXE_NAME}.
|
|
||||||
echo ============================================================
|
|
||||||
echo.
|
|
||||||
pause
|
|
||||||
|
|
||||||
echo Cleaning up helper files...
|
|
||||||
del "{pid_file_path}"
|
|
||||||
del "%~f0"
|
|
||||||
ENDLOCAL
|
|
||||||
"""
|
|
||||||
with open(updater_script_path, "w") as f:
|
|
||||||
f.write(script_content)
|
|
||||||
|
|
||||||
# --- Go back to the os.startfile command that we know works ---
|
|
||||||
os.startfile(updater_script_path)
|
|
||||||
|
|
||||||
self.download_finished.emit()
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
self.download_error.emit(f"Failed to download or run updater: {e}")
|
|
||||||
@@ -3,7 +3,7 @@ import html
|
|||||||
import re
|
import re
|
||||||
|
|
||||||
# --- Third-Party Library Imports ---
|
# --- Third-Party Library Imports ---
|
||||||
import cloudscraper # MODIFIED: Import cloudscraper
|
import requests
|
||||||
from PyQt5.QtCore import QCoreApplication, Qt
|
from PyQt5.QtCore import QCoreApplication, Qt
|
||||||
from PyQt5.QtWidgets import (
|
from PyQt5.QtWidgets import (
|
||||||
QApplication, QDialog, QHBoxLayout, QLabel, QLineEdit, QListWidget,
|
QApplication, QDialog, QHBoxLayout, QLabel, QLineEdit, QListWidget,
|
||||||
@@ -12,6 +12,7 @@ from PyQt5.QtWidgets import (
|
|||||||
|
|
||||||
# --- Local Application Imports ---
|
# --- Local Application Imports ---
|
||||||
from ...i18n.translator import get_translation
|
from ...i18n.translator import get_translation
|
||||||
|
# Corrected Import: Get the icon from the new assets utility module
|
||||||
from ..assets import get_app_icon_object
|
from ..assets import get_app_icon_object
|
||||||
from ...utils.network_utils import prepare_cookies_for_request
|
from ...utils.network_utils import prepare_cookies_for_request
|
||||||
from .CookieHelpDialog import CookieHelpDialog
|
from .CookieHelpDialog import CookieHelpDialog
|
||||||
@@ -40,9 +41,9 @@ class FavoriteArtistsDialog (QDialog ):
|
|||||||
service_lower = service_name.lower()
|
service_lower = service_name.lower()
|
||||||
coomer_primary_services = {'onlyfans', 'fansly', 'manyvids', 'candfans'}
|
coomer_primary_services = {'onlyfans', 'fansly', 'manyvids', 'candfans'}
|
||||||
if service_lower in coomer_primary_services:
|
if service_lower in coomer_primary_services:
|
||||||
return "coomer.st"
|
return "coomer.st" # Use the new domain
|
||||||
else:
|
else:
|
||||||
return "kemono.cr"
|
return "kemono.cr" # Use the new domain
|
||||||
|
|
||||||
def _tr (self ,key ,default_text =""):
|
def _tr (self ,key ,default_text =""):
|
||||||
"""Helper to get translation based on current app language."""
|
"""Helper to get translation based on current app language."""
|
||||||
@@ -125,11 +126,9 @@ class FavoriteArtistsDialog (QDialog ):
|
|||||||
self .artist_list_widget .setVisible (show )
|
self .artist_list_widget .setVisible (show )
|
||||||
|
|
||||||
def _fetch_favorite_artists (self ):
|
def _fetch_favorite_artists (self ):
|
||||||
# --- FIX: Use cloudscraper and add proper headers ---
|
|
||||||
scraper = cloudscraper.create_scraper()
|
|
||||||
# --- END FIX ---
|
|
||||||
|
|
||||||
if self.cookies_config['use_cookie']:
|
if self.cookies_config['use_cookie']:
|
||||||
|
# --- Kemono Check with Fallback ---
|
||||||
kemono_cookies = prepare_cookies_for_request(
|
kemono_cookies = prepare_cookies_for_request(
|
||||||
True, self.cookies_config['cookie_text'], self.cookies_config['selected_cookie_file'],
|
True, self.cookies_config['cookie_text'], self.cookies_config['selected_cookie_file'],
|
||||||
self.cookies_config['app_base_dir'], self._logger, target_domain="kemono.cr"
|
self.cookies_config['app_base_dir'], self._logger, target_domain="kemono.cr"
|
||||||
@@ -141,6 +140,7 @@ class FavoriteArtistsDialog (QDialog ):
|
|||||||
self.cookies_config['app_base_dir'], self._logger, target_domain="kemono.su"
|
self.cookies_config['app_base_dir'], self._logger, target_domain="kemono.su"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# --- Coomer Check with Fallback ---
|
||||||
coomer_cookies = prepare_cookies_for_request(
|
coomer_cookies = prepare_cookies_for_request(
|
||||||
True, self.cookies_config['cookie_text'], self.cookies_config['selected_cookie_file'],
|
True, self.cookies_config['cookie_text'], self.cookies_config['selected_cookie_file'],
|
||||||
self.cookies_config['app_base_dir'], self._logger, target_domain="coomer.st"
|
self.cookies_config['app_base_dir'], self._logger, target_domain="coomer.st"
|
||||||
@@ -153,21 +153,28 @@ class FavoriteArtistsDialog (QDialog ):
|
|||||||
)
|
)
|
||||||
|
|
||||||
if not kemono_cookies and not coomer_cookies:
|
if not kemono_cookies and not coomer_cookies:
|
||||||
|
# If cookies are enabled but none could be loaded, show help and stop.
|
||||||
self.status_label.setText(self._tr("fav_artists_cookies_required_status", "Error: Cookies enabled but could not be loaded for any source."))
|
self.status_label.setText(self._tr("fav_artists_cookies_required_status", "Error: Cookies enabled but could not be loaded for any source."))
|
||||||
self._logger("Error: Cookies enabled but no valid cookies were loaded. Showing help dialog.")
|
self._logger("Error: Cookies enabled but no valid cookies were loaded. Showing help dialog.")
|
||||||
cookie_help_dialog = CookieHelpDialog(self.parent_app, self)
|
cookie_help_dialog = CookieHelpDialog(self.parent_app, self)
|
||||||
cookie_help_dialog.exec_()
|
cookie_help_dialog.exec_()
|
||||||
self.download_button.setEnabled(False)
|
self.download_button.setEnabled(False)
|
||||||
return
|
return # Stop further execution
|
||||||
|
|
||||||
|
kemono_fav_url ="https://kemono.su/api/v1/account/favorites?type=artist"
|
||||||
|
coomer_fav_url ="https://coomer.su/api/v1/account/favorites?type=artist"
|
||||||
|
|
||||||
self .all_fetched_artists =[]
|
self .all_fetched_artists =[]
|
||||||
fetched_any_successfully =False
|
fetched_any_successfully =False
|
||||||
errors_occurred =[]
|
errors_occurred =[]
|
||||||
any_cookies_loaded_successfully_for_any_source =False
|
any_cookies_loaded_successfully_for_any_source =False
|
||||||
|
|
||||||
|
kemono_cr_fav_url = "https://kemono.cr/api/v1/account/favorites?type=artist"
|
||||||
|
coomer_st_fav_url = "https://coomer.st/api/v1/account/favorites?type=artist"
|
||||||
|
|
||||||
api_sources = [
|
api_sources = [
|
||||||
{"name": "Kemono.cr", "url": "https://kemono.cr/api/v1/account/favorites?type=artist", "domain": "kemono.cr"},
|
{"name": "Kemono.cr", "url": kemono_cr_fav_url, "domain": "kemono.cr"},
|
||||||
{"name": "Coomer.st", "url": "https://coomer.st/api/v1/account/favorites?type=artist", "domain": "coomer.st"}
|
{"name": "Coomer.st", "url": coomer_st_fav_url, "domain": "coomer.st"}
|
||||||
]
|
]
|
||||||
|
|
||||||
for source in api_sources :
|
for source in api_sources :
|
||||||
@@ -178,36 +185,41 @@ class FavoriteArtistsDialog (QDialog ):
|
|||||||
cookies_dict_for_source = None
|
cookies_dict_for_source = None
|
||||||
if self.cookies_config['use_cookie']:
|
if self.cookies_config['use_cookie']:
|
||||||
primary_domain = source['domain']
|
primary_domain = source['domain']
|
||||||
fallback_domain = "kemono.su" if "kemono" in primary_domain else "coomer.su"
|
fallback_domain = None
|
||||||
|
if primary_domain == "kemono.cr":
|
||||||
|
fallback_domain = "kemono.su"
|
||||||
|
elif primary_domain == "coomer.st":
|
||||||
|
fallback_domain = "coomer.su"
|
||||||
|
|
||||||
|
# First, try the primary domain
|
||||||
cookies_dict_for_source = prepare_cookies_for_request(
|
cookies_dict_for_source = prepare_cookies_for_request(
|
||||||
True, self.cookies_config['cookie_text'], self.cookies_config['selected_cookie_file'],
|
True,
|
||||||
self.cookies_config['app_base_dir'], self._logger, target_domain=primary_domain
|
self.cookies_config['cookie_text'],
|
||||||
|
self.cookies_config['selected_cookie_file'],
|
||||||
|
self.cookies_config['app_base_dir'],
|
||||||
|
self._logger,
|
||||||
|
target_domain=primary_domain
|
||||||
)
|
)
|
||||||
if not cookies_dict_for_source:
|
|
||||||
self._logger(f"Warning ({source['name']}): No cookies for '{primary_domain}'. Trying fallback '{fallback_domain}'...")
|
# If no cookies found, try the fallback domain
|
||||||
|
if not cookies_dict_for_source and fallback_domain:
|
||||||
|
self._logger(f"Warning ({source['name']}): No cookies found for '{primary_domain}'. Trying fallback '{fallback_domain}'...")
|
||||||
cookies_dict_for_source = prepare_cookies_for_request(
|
cookies_dict_for_source = prepare_cookies_for_request(
|
||||||
True, self.cookies_config['cookie_text'], self.cookies_config['selected_cookie_file'],
|
True,
|
||||||
self.cookies_config['app_base_dir'], self._logger, target_domain=fallback_domain
|
self.cookies_config['cookie_text'],
|
||||||
|
self.cookies_config['selected_cookie_file'],
|
||||||
|
self.cookies_config['app_base_dir'],
|
||||||
|
self._logger,
|
||||||
|
target_domain=fallback_domain
|
||||||
)
|
)
|
||||||
|
|
||||||
if cookies_dict_for_source:
|
if cookies_dict_for_source:
|
||||||
any_cookies_loaded_successfully_for_any_source = True
|
any_cookies_loaded_successfully_for_any_source = True
|
||||||
else:
|
else:
|
||||||
self._logger(f"Warning ({source['name']}): Cookies enabled but not loaded for this source. Fetch may fail.")
|
self._logger(f"Warning ({source['name']}): Cookies enabled but could not be loaded for this source (including fallbacks). Fetch might fail.")
|
||||||
try :
|
try :
|
||||||
# --- FIX: Add Referer and Accept headers ---
|
headers ={'User-Agent':'Mozilla/5.0'}
|
||||||
headers = {
|
response =requests .get (source ['url'],headers =headers ,cookies =cookies_dict_for_source ,timeout =20 )
|
||||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
|
||||||
'Referer': f"https://{source['domain']}/favorites",
|
|
||||||
'Accept': 'text/css'
|
|
||||||
}
|
|
||||||
# --- END FIX ---
|
|
||||||
|
|
||||||
# --- FIX: Use scraper instead of requests ---
|
|
||||||
response = scraper.get(source['url'], headers=headers, cookies=cookies_dict_for_source, timeout=20)
|
|
||||||
# --- END FIX ---
|
|
||||||
|
|
||||||
response .raise_for_status ()
|
response .raise_for_status ()
|
||||||
artists_data_from_api =response .json ()
|
artists_data_from_api =response .json ()
|
||||||
|
|
||||||
@@ -242,10 +254,15 @@ class FavoriteArtistsDialog (QDialog ):
|
|||||||
fetched_any_successfully =True
|
fetched_any_successfully =True
|
||||||
self ._logger (f"Fetched {processed_artists_from_source } artists from {source ['name']}.")
|
self ._logger (f"Fetched {processed_artists_from_source } artists from {source ['name']}.")
|
||||||
|
|
||||||
except Exception as e :
|
except requests .exceptions .RequestException as e :
|
||||||
error_msg =f"Error fetching favorites from {source ['name']}: {e }"
|
error_msg =f"Error fetching favorites from {source ['name']}: {e }"
|
||||||
self ._logger (error_msg )
|
self ._logger (error_msg )
|
||||||
errors_occurred .append (error_msg )
|
errors_occurred .append (error_msg )
|
||||||
|
except Exception as e :
|
||||||
|
error_msg =f"An unexpected error occurred with {source ['name']}: {e }"
|
||||||
|
self ._logger (error_msg )
|
||||||
|
errors_occurred .append (error_msg )
|
||||||
|
|
||||||
|
|
||||||
if self .cookies_config ['use_cookie']and not any_cookies_loaded_successfully_for_any_source :
|
if self .cookies_config ['use_cookie']and not any_cookies_loaded_successfully_for_any_source :
|
||||||
self .status_label .setText (self ._tr ("fav_artists_cookies_required_status","Error: Cookies enabled but could not be loaded for any source."))
|
self .status_label .setText (self ._tr ("fav_artists_cookies_required_status","Error: Cookies enabled but could not be loaded for any source."))
|
||||||
@@ -271,7 +288,7 @@ class FavoriteArtistsDialog (QDialog ):
|
|||||||
self ._show_content_elements (True )
|
self ._show_content_elements (True )
|
||||||
self .download_button .setEnabled (True )
|
self .download_button .setEnabled (True )
|
||||||
elif not fetched_any_successfully and not errors_occurred :
|
elif not fetched_any_successfully and not errors_occurred :
|
||||||
self .status_label .setText (self ._tr ("fav_artists_none_found_status","No favorite artists found on Kemono or Coomer."))
|
self .status_label .setText (self ._tr ("fav_artists_none_found_status","No favorite artists found on Kemono.su or Coomer.su."))
|
||||||
self ._show_content_elements (False )
|
self ._show_content_elements (False )
|
||||||
self .download_button .setEnabled (False )
|
self .download_button .setEnabled (False )
|
||||||
else :
|
else :
|
||||||
@@ -327,4 +344,4 @@ class FavoriteArtistsDialog (QDialog ):
|
|||||||
self .accept ()
|
self .accept ()
|
||||||
|
|
||||||
def get_selected_artists (self ):
|
def get_selected_artists (self ):
|
||||||
return self .selected_artists_data
|
return self .selected_artists_data
|
||||||
@@ -7,7 +7,7 @@ import traceback
|
|||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
import cloudscraper # MODIFIED: Import cloudscraper
|
import requests
|
||||||
from PyQt5.QtCore import QCoreApplication, Qt, pyqtSignal, QThread
|
from PyQt5.QtCore import QCoreApplication, Qt, pyqtSignal, QThread
|
||||||
from PyQt5.QtWidgets import (
|
from PyQt5.QtWidgets import (
|
||||||
QApplication, QDialog, QHBoxLayout, QLabel, QLineEdit, QListWidget,
|
QApplication, QDialog, QHBoxLayout, QLabel, QLineEdit, QListWidget,
|
||||||
@@ -42,9 +42,10 @@ class FavoritePostsFetcherThread (QThread ):
|
|||||||
self .parent_logger_func (f"[FavPostsFetcherThread] {message }")
|
self .parent_logger_func (f"[FavPostsFetcherThread] {message }")
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
# --- FIX: Use cloudscraper and add proper headers ---
|
kemono_su_fav_posts_url = "https://kemono.su/api/v1/account/favorites?type=post"
|
||||||
scraper = cloudscraper.create_scraper()
|
coomer_su_fav_posts_url = "https://coomer.su/api/v1/account/favorites?type=post"
|
||||||
# --- END FIX ---
|
kemono_cr_fav_posts_url = "https://kemono.cr/api/v1/account/favorites?type=post"
|
||||||
|
coomer_st_fav_posts_url = "https://coomer.st/api/v1/account/favorites?type=post"
|
||||||
|
|
||||||
all_fetched_posts_temp = []
|
all_fetched_posts_temp = []
|
||||||
error_messages_for_summary = []
|
error_messages_for_summary = []
|
||||||
@@ -55,8 +56,8 @@ class FavoritePostsFetcherThread (QThread ):
|
|||||||
self.progress_bar_update.emit(0, 0)
|
self.progress_bar_update.emit(0, 0)
|
||||||
|
|
||||||
api_sources = [
|
api_sources = [
|
||||||
{"name": "Kemono.cr", "url": "https://kemono.cr/api/v1/account/favorites?type=post", "domain": "kemono.cr"},
|
{"name": "Kemono.cr", "url": kemono_cr_fav_posts_url, "domain": "kemono.cr"},
|
||||||
{"name": "Coomer.st", "url": "https://coomer.st/api/v1/account/favorites?type=post", "domain": "coomer.st"}
|
{"name": "Coomer.st", "url": coomer_st_fav_posts_url, "domain": "coomer.st"}
|
||||||
]
|
]
|
||||||
|
|
||||||
api_sources_to_try =[]
|
api_sources_to_try =[]
|
||||||
@@ -80,18 +81,32 @@ class FavoritePostsFetcherThread (QThread ):
|
|||||||
cookies_dict_for_source = None
|
cookies_dict_for_source = None
|
||||||
if self.cookies_config['use_cookie']:
|
if self.cookies_config['use_cookie']:
|
||||||
primary_domain = source['domain']
|
primary_domain = source['domain']
|
||||||
fallback_domain = "kemono.su" if "kemono" in primary_domain else "coomer.su"
|
fallback_domain = None
|
||||||
|
if primary_domain == "kemono.cr":
|
||||||
|
fallback_domain = "kemono.su"
|
||||||
|
elif primary_domain == "coomer.st":
|
||||||
|
fallback_domain = "coomer.su"
|
||||||
|
|
||||||
|
# First, try the primary domain
|
||||||
cookies_dict_for_source = prepare_cookies_for_request(
|
cookies_dict_for_source = prepare_cookies_for_request(
|
||||||
True, self.cookies_config['cookie_text'], self.cookies_config['selected_cookie_file'],
|
True,
|
||||||
self.cookies_config['app_base_dir'], self._logger, target_domain=primary_domain
|
self.cookies_config['cookie_text'],
|
||||||
|
self.cookies_config['selected_cookie_file'],
|
||||||
|
self.cookies_config['app_base_dir'],
|
||||||
|
self._logger,
|
||||||
|
target_domain=primary_domain
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# If no cookies found, try the fallback domain
|
||||||
if not cookies_dict_for_source and fallback_domain:
|
if not cookies_dict_for_source and fallback_domain:
|
||||||
self._logger(f"Warning ({source['name']}): No cookies for '{primary_domain}'. Trying fallback '{fallback_domain}'...")
|
self._logger(f"Warning ({source['name']}): No cookies found for '{primary_domain}'. Trying fallback '{fallback_domain}'...")
|
||||||
cookies_dict_for_source = prepare_cookies_for_request(
|
cookies_dict_for_source = prepare_cookies_for_request(
|
||||||
True, self.cookies_config['cookie_text'], self.cookies_config['selected_cookie_file'],
|
True,
|
||||||
self.cookies_config['app_base_dir'], self._logger, target_domain=fallback_domain
|
self.cookies_config['cookie_text'],
|
||||||
|
self.cookies_config['selected_cookie_file'],
|
||||||
|
self.cookies_config['app_base_dir'],
|
||||||
|
self._logger,
|
||||||
|
target_domain=fallback_domain
|
||||||
)
|
)
|
||||||
|
|
||||||
if cookies_dict_for_source:
|
if cookies_dict_for_source:
|
||||||
@@ -105,18 +120,8 @@ class FavoritePostsFetcherThread (QThread ):
|
|||||||
QCoreApplication .processEvents ()
|
QCoreApplication .processEvents ()
|
||||||
|
|
||||||
try :
|
try :
|
||||||
# --- FIX: Add Referer and Accept headers ---
|
headers ={'User-Agent':'Mozilla/5.0'}
|
||||||
headers = {
|
response =requests .get (source ['url'],headers =headers ,cookies =cookies_dict_for_source ,timeout =20 )
|
||||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
|
|
||||||
'Referer': f"https://{source['domain']}/favorites",
|
|
||||||
'Accept': 'text/css'
|
|
||||||
}
|
|
||||||
# --- END FIX ---
|
|
||||||
|
|
||||||
# --- FIX: Use scraper instead of requests ---
|
|
||||||
response = scraper.get(source['url'], headers=headers, cookies=cookies_dict_for_source, timeout=20)
|
|
||||||
# --- END FIX ---
|
|
||||||
|
|
||||||
response .raise_for_status ()
|
response .raise_for_status ()
|
||||||
posts_data_from_api =response .json ()
|
posts_data_from_api =response .json ()
|
||||||
|
|
||||||
@@ -148,24 +153,33 @@ class FavoritePostsFetcherThread (QThread ):
|
|||||||
fetched_any_successfully =True
|
fetched_any_successfully =True
|
||||||
self ._logger (f"Fetched {processed_posts_from_source } posts from {source ['name']}.")
|
self ._logger (f"Fetched {processed_posts_from_source } posts from {source ['name']}.")
|
||||||
|
|
||||||
except Exception as e :
|
except requests .exceptions .RequestException as e :
|
||||||
err_detail =f"Error fetching favorite posts from {source ['name']}: {e }"
|
err_detail =f"Error fetching favorite posts from {source ['name']}: {e }"
|
||||||
self ._logger (err_detail )
|
self ._logger (err_detail )
|
||||||
error_messages_for_summary .append (err_detail )
|
error_messages_for_summary .append (err_detail )
|
||||||
if hasattr(e, 'response') and e.response is not None and e.response.status_code == 401:
|
if e .response is not None and e .response .status_code ==401 :
|
||||||
self .finished .emit ([],"KEY_AUTH_FAILED")
|
self .finished .emit ([],"KEY_AUTH_FAILED")
|
||||||
self ._logger (f"Authorization failed for {source ['name']}, emitting KEY_AUTH_FAILED.")
|
self ._logger (f"Authorization failed for {source ['name']}, emitting KEY_AUTH_FAILED.")
|
||||||
return
|
return
|
||||||
|
except Exception as e :
|
||||||
|
err_detail =f"An unexpected error occurred with {source ['name']}: {e }"
|
||||||
|
self ._logger (err_detail )
|
||||||
|
error_messages_for_summary .append (err_detail )
|
||||||
|
|
||||||
if self .cancellation_event .is_set ():
|
if self .cancellation_event .is_set ():
|
||||||
self .finished .emit ([],"KEY_FETCH_CANCELLED_AFTER")
|
self .finished .emit ([],"KEY_FETCH_CANCELLED_AFTER")
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
if self .cookies_config ['use_cookie']and not any_cookies_loaded_successfully_for_any_source :
|
if self .cookies_config ['use_cookie']and not any_cookies_loaded_successfully_for_any_source :
|
||||||
|
|
||||||
if self .target_domain_preference and not any_cookies_loaded_successfully_for_any_source :
|
if self .target_domain_preference and not any_cookies_loaded_successfully_for_any_source :
|
||||||
|
|
||||||
domain_key_part =self .error_key_map .get (self .target_domain_preference ,self .target_domain_preference .lower ().replace ('.','_'))
|
domain_key_part =self .error_key_map .get (self .target_domain_preference ,self .target_domain_preference .lower ().replace ('.','_'))
|
||||||
self .finished .emit ([],f"KEY_COOKIES_REQUIRED_BUT_NOT_FOUND_FOR_DOMAIN_{domain_key_part }")
|
self .finished .emit ([],f"KEY_COOKIES_REQUIRED_BUT_NOT_FOUND_FOR_DOMAIN_{domain_key_part }")
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
self .finished .emit ([],"KEY_COOKIES_REQUIRED_BUT_NOT_FOUND_GENERIC")
|
self .finished .emit ([],"KEY_COOKIES_REQUIRED_BUT_NOT_FOUND_GENERIC")
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -629,4 +643,4 @@ class FavoritePostsDialog (QDialog ):
|
|||||||
self .accept ()
|
self .accept ()
|
||||||
|
|
||||||
def get_selected_posts (self ):
|
def get_selected_posts (self ):
|
||||||
return self .selected_posts_data
|
return self .selected_posts_data
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
# --- Standard Library Imports ---
|
# --- Standard Library Imports ---
|
||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
import sys
|
|
||||||
|
|
||||||
# --- PyQt5 Imports ---
|
# --- PyQt5 Imports ---
|
||||||
from PyQt5.QtCore import Qt, QStandardPaths
|
from PyQt5.QtCore import Qt, QStandardPaths
|
||||||
@@ -18,9 +17,9 @@ from ...config.constants import (
|
|||||||
THEME_KEY, LANGUAGE_KEY, DOWNLOAD_LOCATION_KEY,
|
THEME_KEY, LANGUAGE_KEY, DOWNLOAD_LOCATION_KEY,
|
||||||
RESOLUTION_KEY, UI_SCALE_KEY, SAVE_CREATOR_JSON_KEY,
|
RESOLUTION_KEY, UI_SCALE_KEY, SAVE_CREATOR_JSON_KEY,
|
||||||
COOKIE_TEXT_KEY, USE_COOKIE_KEY,
|
COOKIE_TEXT_KEY, USE_COOKIE_KEY,
|
||||||
FETCH_FIRST_KEY
|
FETCH_FIRST_KEY ### ADDED ###
|
||||||
)
|
)
|
||||||
from ...services.updater import UpdateChecker, UpdateDownloader
|
|
||||||
|
|
||||||
class FutureSettingsDialog(QDialog):
|
class FutureSettingsDialog(QDialog):
|
||||||
"""
|
"""
|
||||||
@@ -31,7 +30,6 @@ class FutureSettingsDialog(QDialog):
|
|||||||
super().__init__(parent)
|
super().__init__(parent)
|
||||||
self.parent_app = parent_app_ref
|
self.parent_app = parent_app_ref
|
||||||
self.setModal(True)
|
self.setModal(True)
|
||||||
self.update_downloader_thread = None # To keep a reference
|
|
||||||
|
|
||||||
app_icon = get_app_icon_object()
|
app_icon = get_app_icon_object()
|
||||||
if app_icon and not app_icon.isNull():
|
if app_icon and not app_icon.isNull():
|
||||||
@@ -39,7 +37,7 @@ class FutureSettingsDialog(QDialog):
|
|||||||
|
|
||||||
screen_height = QApplication.primaryScreen().availableGeometry().height() if QApplication.primaryScreen() else 800
|
screen_height = QApplication.primaryScreen().availableGeometry().height() if QApplication.primaryScreen() else 800
|
||||||
scale_factor = screen_height / 800.0
|
scale_factor = screen_height / 800.0
|
||||||
base_min_w, base_min_h = 420, 480 # Increased height for update section
|
base_min_w, base_min_h = 420, 390
|
||||||
scaled_min_w = int(base_min_w * scale_factor)
|
scaled_min_w = int(base_min_w * scale_factor)
|
||||||
scaled_min_h = int(base_min_h * scale_factor)
|
scaled_min_h = int(base_min_h * scale_factor)
|
||||||
self.setMinimumSize(scaled_min_w, scaled_min_h)
|
self.setMinimumSize(scaled_min_w, scaled_min_h)
|
||||||
@@ -55,19 +53,21 @@ class FutureSettingsDialog(QDialog):
|
|||||||
self.interface_group_box = QGroupBox()
|
self.interface_group_box = QGroupBox()
|
||||||
interface_layout = QGridLayout(self.interface_group_box)
|
interface_layout = QGridLayout(self.interface_group_box)
|
||||||
|
|
||||||
# Theme, UI Scale, Language (unchanged)...
|
# Theme
|
||||||
self.theme_label = QLabel()
|
self.theme_label = QLabel()
|
||||||
self.theme_toggle_button = QPushButton()
|
self.theme_toggle_button = QPushButton()
|
||||||
self.theme_toggle_button.clicked.connect(self._toggle_theme)
|
self.theme_toggle_button.clicked.connect(self._toggle_theme)
|
||||||
interface_layout.addWidget(self.theme_label, 0, 0)
|
interface_layout.addWidget(self.theme_label, 0, 0)
|
||||||
interface_layout.addWidget(self.theme_toggle_button, 0, 1)
|
interface_layout.addWidget(self.theme_toggle_button, 0, 1)
|
||||||
|
|
||||||
|
# UI Scale
|
||||||
self.ui_scale_label = QLabel()
|
self.ui_scale_label = QLabel()
|
||||||
self.ui_scale_combo_box = QComboBox()
|
self.ui_scale_combo_box = QComboBox()
|
||||||
self.ui_scale_combo_box.currentIndexChanged.connect(self._display_setting_changed)
|
self.ui_scale_combo_box.currentIndexChanged.connect(self._display_setting_changed)
|
||||||
interface_layout.addWidget(self.ui_scale_label, 1, 0)
|
interface_layout.addWidget(self.ui_scale_label, 1, 0)
|
||||||
interface_layout.addWidget(self.ui_scale_combo_box, 1, 1)
|
interface_layout.addWidget(self.ui_scale_combo_box, 1, 1)
|
||||||
|
|
||||||
|
# Language
|
||||||
self.language_label = QLabel()
|
self.language_label = QLabel()
|
||||||
self.language_combo_box = QComboBox()
|
self.language_combo_box = QComboBox()
|
||||||
self.language_combo_box.currentIndexChanged.connect(self._language_selection_changed)
|
self.language_combo_box.currentIndexChanged.connect(self._language_selection_changed)
|
||||||
@@ -78,7 +78,6 @@ class FutureSettingsDialog(QDialog):
|
|||||||
|
|
||||||
self.download_window_group_box = QGroupBox()
|
self.download_window_group_box = QGroupBox()
|
||||||
download_window_layout = QGridLayout(self.download_window_group_box)
|
download_window_layout = QGridLayout(self.download_window_group_box)
|
||||||
|
|
||||||
self.window_size_label = QLabel()
|
self.window_size_label = QLabel()
|
||||||
self.resolution_combo_box = QComboBox()
|
self.resolution_combo_box = QComboBox()
|
||||||
self.resolution_combo_box.currentIndexChanged.connect(self._display_setting_changed)
|
self.resolution_combo_box.currentIndexChanged.connect(self._display_setting_changed)
|
||||||
@@ -92,7 +91,7 @@ class FutureSettingsDialog(QDialog):
|
|||||||
download_window_layout.addWidget(self.save_path_button, 1, 1)
|
download_window_layout.addWidget(self.save_path_button, 1, 1)
|
||||||
|
|
||||||
self.save_creator_json_checkbox = QCheckBox()
|
self.save_creator_json_checkbox = QCheckBox()
|
||||||
self.save_creator_json_checkbox.stateChanged.connect(self._creator_json_setting_changed)
|
self.save_creator_json_checkbox.stateChanged.connect(self._creator_json_setting_changed)
|
||||||
download_window_layout.addWidget(self.save_creator_json_checkbox, 2, 0, 1, 2)
|
download_window_layout.addWidget(self.save_creator_json_checkbox, 2, 0, 1, 2)
|
||||||
|
|
||||||
self.fetch_first_checkbox = QCheckBox()
|
self.fetch_first_checkbox = QCheckBox()
|
||||||
@@ -101,96 +100,14 @@ class FutureSettingsDialog(QDialog):
|
|||||||
|
|
||||||
main_layout.addWidget(self.download_window_group_box)
|
main_layout.addWidget(self.download_window_group_box)
|
||||||
|
|
||||||
# --- NEW: Update Section ---
|
|
||||||
self.update_group_box = QGroupBox()
|
|
||||||
update_layout = QGridLayout(self.update_group_box)
|
|
||||||
self.version_label = QLabel()
|
|
||||||
self.update_status_label = QLabel()
|
|
||||||
self.check_update_button = QPushButton()
|
|
||||||
self.check_update_button.clicked.connect(self._check_for_updates)
|
|
||||||
update_layout.addWidget(self.version_label, 0, 0)
|
|
||||||
update_layout.addWidget(self.update_status_label, 0, 1)
|
|
||||||
update_layout.addWidget(self.check_update_button, 1, 0, 1, 2)
|
|
||||||
main_layout.addWidget(self.update_group_box)
|
|
||||||
# --- END: New Section ---
|
|
||||||
|
|
||||||
main_layout.addStretch(1)
|
main_layout.addStretch(1)
|
||||||
|
|
||||||
self.ok_button = QPushButton()
|
self.ok_button = QPushButton()
|
||||||
self.ok_button.clicked.connect(self.accept)
|
self.ok_button.clicked.connect(self.accept)
|
||||||
main_layout.addWidget(self.ok_button, 0, Qt.AlignRight | Qt.AlignBottom)
|
main_layout.addWidget(self.ok_button, 0, Qt.AlignRight | Qt.AlignBottom)
|
||||||
|
|
||||||
def _retranslate_ui(self):
|
|
||||||
self.setWindowTitle(self._tr("settings_dialog_title", "Settings"))
|
|
||||||
self.interface_group_box.setTitle(self._tr("interface_group_title", "Interface Settings"))
|
|
||||||
self.download_window_group_box.setTitle(self._tr("download_window_group_title", "Download & Window Settings"))
|
|
||||||
self.theme_label.setText(self._tr("theme_label", "Theme:"))
|
|
||||||
self.ui_scale_label.setText(self._tr("ui_scale_label", "UI Scale:"))
|
|
||||||
self.language_label.setText(self._tr("language_label", "Language:"))
|
|
||||||
self.window_size_label.setText(self._tr("window_size_label", "Window Size:"))
|
|
||||||
self.default_path_label.setText(self._tr("default_path_label", "Default Path:"))
|
|
||||||
self.save_creator_json_checkbox.setText(self._tr("save_creator_json_label", "Save Creator.json file"))
|
|
||||||
self.fetch_first_checkbox.setText(self._tr("fetch_first_label", "Fetch First (Download after all pages are found)"))
|
|
||||||
self.fetch_first_checkbox.setToolTip(self._tr("fetch_first_tooltip", "If checked, the downloader will find all posts from a creator first before starting any downloads.\nThis can be slower to start but provides a more accurate progress bar."))
|
|
||||||
self._update_theme_toggle_button_text()
|
|
||||||
self.save_path_button.setText(self._tr("settings_save_cookie_path_button", "Save Cookie + Download Path"))
|
|
||||||
self.save_path_button.setToolTip(self._tr("settings_save_cookie_path_tooltip", "Save the current 'Download Location' and Cookie settings for future sessions."))
|
|
||||||
self.ok_button.setText(self._tr("ok_button", "OK"))
|
|
||||||
|
|
||||||
# --- NEW: Translations for Update Section ---
|
|
||||||
self.update_group_box.setTitle(self._tr("update_group_title", "Application Updates"))
|
|
||||||
current_version = self.parent_app.windowTitle().split(' v')[-1]
|
|
||||||
self.version_label.setText(self._tr("current_version_label", f"Current Version: v{current_version}"))
|
|
||||||
self.update_status_label.setText(self._tr("update_status_ready", "Ready to check."))
|
|
||||||
self.check_update_button.setText(self._tr("check_for_updates_button", "Check for Updates"))
|
|
||||||
# --- END: New Translations ---
|
|
||||||
|
|
||||||
self._populate_display_combo_boxes()
|
|
||||||
self._populate_language_combo_box()
|
|
||||||
self._load_checkbox_states()
|
|
||||||
|
|
||||||
def _check_for_updates(self):
|
|
||||||
"""Starts the update check thread."""
|
|
||||||
self.check_update_button.setEnabled(False)
|
|
||||||
self.update_status_label.setText(self._tr("update_status_checking", "Checking..."))
|
|
||||||
current_version = self.parent_app.windowTitle().split(' v')[-1]
|
|
||||||
|
|
||||||
self.update_checker_thread = UpdateChecker(current_version)
|
|
||||||
self.update_checker_thread.update_available.connect(self._on_update_available)
|
|
||||||
self.update_checker_thread.up_to_date.connect(self._on_up_to_date)
|
|
||||||
self.update_checker_thread.update_error.connect(self._on_update_error)
|
|
||||||
self.update_checker_thread.start()
|
|
||||||
|
|
||||||
def _on_update_available(self, new_version, download_url):
|
|
||||||
self.update_status_label.setText(self._tr("update_status_found", f"Update found: v{new_version}"))
|
|
||||||
self.check_update_button.setEnabled(True)
|
|
||||||
|
|
||||||
reply = QMessageBox.question(self, self._tr("update_available_title", "Update Available"),
|
|
||||||
self._tr("update_available_message", f"A new version (v{new_version}) is available.\nWould you like to download and install it now?"),
|
|
||||||
QMessageBox.Yes | QMessageBox.No, QMessageBox.Yes)
|
|
||||||
if reply == QMessageBox.Yes:
|
|
||||||
self.ok_button.setEnabled(False)
|
|
||||||
self.check_update_button.setEnabled(False)
|
|
||||||
self.update_status_label.setText(self._tr("update_status_downloading", "Downloading update..."))
|
|
||||||
self.update_downloader_thread = UpdateDownloader(download_url, self.parent_app)
|
|
||||||
self.update_downloader_thread.download_finished.connect(self._on_download_finished)
|
|
||||||
self.update_downloader_thread.download_error.connect(self._on_update_error)
|
|
||||||
self.update_downloader_thread.start()
|
|
||||||
|
|
||||||
def _on_download_finished(self):
|
|
||||||
QApplication.instance().quit()
|
|
||||||
|
|
||||||
def _on_up_to_date(self, message):
|
|
||||||
self.update_status_label.setText(self._tr("update_status_latest", message))
|
|
||||||
self.check_update_button.setEnabled(True)
|
|
||||||
|
|
||||||
def _on_update_error(self, message):
|
|
||||||
self.update_status_label.setText(self._tr("update_status_error", f"Error: {message}"))
|
|
||||||
self.check_update_button.setEnabled(True)
|
|
||||||
self.ok_button.setEnabled(True)
|
|
||||||
|
|
||||||
# --- (The rest of the file remains unchanged from your provided code) ---
|
|
||||||
def _load_checkbox_states(self):
|
def _load_checkbox_states(self):
|
||||||
|
"""Loads the initial state for all checkboxes from settings."""
|
||||||
self.save_creator_json_checkbox.blockSignals(True)
|
self.save_creator_json_checkbox.blockSignals(True)
|
||||||
should_save = self.parent_app.settings.value(SAVE_CREATOR_JSON_KEY, True, type=bool)
|
should_save = self.parent_app.settings.value(SAVE_CREATOR_JSON_KEY, True, type=bool)
|
||||||
self.save_creator_json_checkbox.setChecked(should_save)
|
self.save_creator_json_checkbox.setChecked(should_save)
|
||||||
@@ -202,11 +119,13 @@ class FutureSettingsDialog(QDialog):
|
|||||||
self.fetch_first_checkbox.blockSignals(False)
|
self.fetch_first_checkbox.blockSignals(False)
|
||||||
|
|
||||||
def _creator_json_setting_changed(self, state):
|
def _creator_json_setting_changed(self, state):
|
||||||
|
"""Saves the state of the 'Save Creator.json' checkbox."""
|
||||||
is_checked = state == Qt.Checked
|
is_checked = state == Qt.Checked
|
||||||
self.parent_app.settings.setValue(SAVE_CREATOR_JSON_KEY, is_checked)
|
self.parent_app.settings.setValue(SAVE_CREATOR_JSON_KEY, is_checked)
|
||||||
self.parent_app.settings.sync()
|
self.parent_app.settings.sync()
|
||||||
|
|
||||||
def _fetch_first_setting_changed(self, state):
|
def _fetch_first_setting_changed(self, state):
|
||||||
|
"""Saves the state of the 'Fetch First' checkbox."""
|
||||||
is_checked = state == Qt.Checked
|
is_checked = state == Qt.Checked
|
||||||
self.parent_app.settings.setValue(FETCH_FIRST_KEY, is_checked)
|
self.parent_app.settings.setValue(FETCH_FIRST_KEY, is_checked)
|
||||||
self.parent_app.settings.sync()
|
self.parent_app.settings.sync()
|
||||||
@@ -216,6 +135,34 @@ class FutureSettingsDialog(QDialog):
|
|||||||
return get_translation(self.parent_app.current_selected_language, key, default_text)
|
return get_translation(self.parent_app.current_selected_language, key, default_text)
|
||||||
return default_text
|
return default_text
|
||||||
|
|
||||||
|
def _retranslate_ui(self):
|
||||||
|
self.setWindowTitle(self._tr("settings_dialog_title", "Settings"))
|
||||||
|
|
||||||
|
self.interface_group_box.setTitle(self._tr("interface_group_title", "Interface Settings"))
|
||||||
|
self.download_window_group_box.setTitle(self._tr("download_window_group_title", "Download & Window Settings"))
|
||||||
|
|
||||||
|
self.theme_label.setText(self._tr("theme_label", "Theme:"))
|
||||||
|
self.ui_scale_label.setText(self._tr("ui_scale_label", "UI Scale:"))
|
||||||
|
self.language_label.setText(self._tr("language_label", "Language:"))
|
||||||
|
|
||||||
|
self.window_size_label.setText(self._tr("window_size_label", "Window Size:"))
|
||||||
|
self.default_path_label.setText(self._tr("default_path_label", "Default Path:"))
|
||||||
|
self.save_creator_json_checkbox.setText(self._tr("save_creator_json_label", "Save Creator.json file"))
|
||||||
|
|
||||||
|
self.fetch_first_checkbox.setText(self._tr("fetch_first_label", "Fetch First (Download after all pages are found)"))
|
||||||
|
self.fetch_first_checkbox.setToolTip(self._tr("fetch_first_tooltip", "If checked, the downloader will find all posts from a creator first before starting any downloads.\nThis can be slower to start but provides a more accurate progress bar."))
|
||||||
|
|
||||||
|
self._update_theme_toggle_button_text()
|
||||||
|
self.save_path_button.setText(self._tr("settings_save_cookie_path_button", "Save Cookie + Download Path"))
|
||||||
|
self.save_path_button.setToolTip(self._tr("settings_save_cookie_path_tooltip", "Save the current 'Download Location' and Cookie settings for future sessions."))
|
||||||
|
self.ok_button.setText(self._tr("ok_button", "OK"))
|
||||||
|
|
||||||
|
self._populate_display_combo_boxes()
|
||||||
|
self._populate_language_combo_box()
|
||||||
|
self._load_checkbox_states()
|
||||||
|
|
||||||
|
# --- (The rest of the file remains unchanged) ---
|
||||||
|
|
||||||
def _apply_theme(self):
|
def _apply_theme(self):
|
||||||
if self.parent_app and self.parent_app.current_theme == "dark":
|
if self.parent_app and self.parent_app.current_theme == "dark":
|
||||||
scale = getattr(self.parent_app, 'scale_factor', 1)
|
scale = getattr(self.parent_app, 'scale_factor', 1)
|
||||||
@@ -241,7 +188,14 @@ class FutureSettingsDialog(QDialog):
|
|||||||
def _populate_display_combo_boxes(self):
|
def _populate_display_combo_boxes(self):
|
||||||
self.resolution_combo_box.blockSignals(True)
|
self.resolution_combo_box.blockSignals(True)
|
||||||
self.resolution_combo_box.clear()
|
self.resolution_combo_box.clear()
|
||||||
resolutions = [("Auto", "Auto"), ("1280x720", "1280x720"), ("1600x900", "1600x900"), ("1920x1080", "1920x1080")]
|
resolutions = [
|
||||||
|
("Auto", self._tr("auto_resolution", "Auto (System Default)")),
|
||||||
|
("1280x720", "1280 x 720"),
|
||||||
|
("1600x900", "1600 x 900"),
|
||||||
|
("1920x1080", "1920 x 1080 (Full HD)"),
|
||||||
|
("2560x1440", "2560 x 1440 (2K)"),
|
||||||
|
("3840x2160", "3840 x 2160 (4K)")
|
||||||
|
]
|
||||||
current_res = self.parent_app.settings.value(RESOLUTION_KEY, "Auto")
|
current_res = self.parent_app.settings.value(RESOLUTION_KEY, "Auto")
|
||||||
for res_key, res_name in resolutions:
|
for res_key, res_name in resolutions:
|
||||||
self.resolution_combo_box.addItem(res_name, res_key)
|
self.resolution_combo_box.addItem(res_name, res_key)
|
||||||
@@ -260,22 +214,35 @@ class FutureSettingsDialog(QDialog):
|
|||||||
(1.50, "150%"),
|
(1.50, "150%"),
|
||||||
(1.75, "175%"),
|
(1.75, "175%"),
|
||||||
(2.0, "200%")
|
(2.0, "200%")
|
||||||
]
|
]
|
||||||
current_scale = self.parent_app.settings.value(UI_SCALE_KEY, 1.0)
|
|
||||||
|
current_scale = float(self.parent_app.settings.value(UI_SCALE_KEY, 1.0))
|
||||||
for scale_val, scale_name in scales:
|
for scale_val, scale_name in scales:
|
||||||
self.ui_scale_combo_box.addItem(scale_name, scale_val)
|
self.ui_scale_combo_box.addItem(scale_name, scale_val)
|
||||||
if abs(float(current_scale) - scale_val) < 0.01:
|
if abs(current_scale - scale_val) < 0.01:
|
||||||
self.ui_scale_combo_box.setCurrentIndex(self.ui_scale_combo_box.count() - 1)
|
self.ui_scale_combo_box.setCurrentIndex(self.ui_scale_combo_box.count() - 1)
|
||||||
self.ui_scale_combo_box.blockSignals(False)
|
self.ui_scale_combo_box.blockSignals(False)
|
||||||
|
|
||||||
def _display_setting_changed(self):
|
def _display_setting_changed(self):
|
||||||
selected_res = self.resolution_combo_box.currentData()
|
selected_res = self.resolution_combo_box.currentData()
|
||||||
selected_scale = self.ui_scale_combo_box.currentData()
|
selected_scale = self.ui_scale_combo_box.currentData()
|
||||||
|
|
||||||
self.parent_app.settings.setValue(RESOLUTION_KEY, selected_res)
|
self.parent_app.settings.setValue(RESOLUTION_KEY, selected_res)
|
||||||
self.parent_app.settings.setValue(UI_SCALE_KEY, selected_scale)
|
self.parent_app.settings.setValue(UI_SCALE_KEY, selected_scale)
|
||||||
self.parent_app.settings.sync()
|
self.parent_app.settings.sync()
|
||||||
QMessageBox.information(self, self._tr("display_change_title", "Display Settings Changed"),
|
|
||||||
self._tr("language_change_message", "A restart is required..."))
|
msg_box = QMessageBox(self)
|
||||||
|
msg_box.setIcon(QMessageBox.Information)
|
||||||
|
msg_box.setWindowTitle(self._tr("display_change_title", "Display Settings Changed"))
|
||||||
|
msg_box.setText(self._tr("language_change_message", "A restart is required for these changes to take effect."))
|
||||||
|
msg_box.setInformativeText(self._tr("language_change_informative", "Would you like to restart now?"))
|
||||||
|
restart_button = msg_box.addButton(self._tr("restart_now_button", "Restart Now"), QMessageBox.ApplyRole)
|
||||||
|
ok_button = msg_box.addButton(self._tr("ok_button", "OK"), QMessageBox.AcceptRole)
|
||||||
|
msg_box.setDefaultButton(ok_button)
|
||||||
|
msg_box.exec_()
|
||||||
|
|
||||||
|
if msg_box.clickedButton() == restart_button:
|
||||||
|
self.parent_app._request_restart_application()
|
||||||
|
|
||||||
def _populate_language_combo_box(self):
|
def _populate_language_combo_box(self):
|
||||||
self.language_combo_box.blockSignals(True)
|
self.language_combo_box.blockSignals(True)
|
||||||
@@ -285,7 +252,7 @@ class FutureSettingsDialog(QDialog):
|
|||||||
("de", "Deutsch (German)"), ("es", "Español (Spanish)"), ("pt", "Português (Portuguese)"),
|
("de", "Deutsch (German)"), ("es", "Español (Spanish)"), ("pt", "Português (Portuguese)"),
|
||||||
("ru", "Русский (Russian)"), ("zh_CN", "简体中文 (Simplified Chinese)"),
|
("ru", "Русский (Russian)"), ("zh_CN", "简体中文 (Simplified Chinese)"),
|
||||||
("zh_TW", "繁體中文 (Traditional Chinese)"), ("ko", "한국어 (Korean)")
|
("zh_TW", "繁體中文 (Traditional Chinese)"), ("ko", "한국어 (Korean)")
|
||||||
]
|
]
|
||||||
current_lang = self.parent_app.current_selected_language
|
current_lang = self.parent_app.current_selected_language
|
||||||
for lang_code, lang_name in languages:
|
for lang_code, lang_name in languages:
|
||||||
self.language_combo_box.addItem(lang_name, lang_code)
|
self.language_combo_box.addItem(lang_name, lang_code)
|
||||||
@@ -299,32 +266,59 @@ class FutureSettingsDialog(QDialog):
|
|||||||
self.parent_app.settings.setValue(LANGUAGE_KEY, selected_lang_code)
|
self.parent_app.settings.setValue(LANGUAGE_KEY, selected_lang_code)
|
||||||
self.parent_app.settings.sync()
|
self.parent_app.settings.sync()
|
||||||
self.parent_app.current_selected_language = selected_lang_code
|
self.parent_app.current_selected_language = selected_lang_code
|
||||||
|
|
||||||
self._retranslate_ui()
|
self._retranslate_ui()
|
||||||
if hasattr(self.parent_app, '_retranslate_main_ui'):
|
if hasattr(self.parent_app, '_retranslate_main_ui'):
|
||||||
self.parent_app._retranslate_main_ui()
|
self.parent_app._retranslate_main_ui()
|
||||||
QMessageBox.information(self, self._tr("language_change_title", "Language Changed"),
|
|
||||||
self._tr("language_change_message", "A restart is required..."))
|
msg_box = QMessageBox(self)
|
||||||
|
msg_box.setIcon(QMessageBox.Information)
|
||||||
|
msg_box.setWindowTitle(self._tr("language_change_title", "Language Changed"))
|
||||||
|
msg_box.setText(self._tr("language_change_message", "A restart is required..."))
|
||||||
|
msg_box.setInformativeText(self._tr("language_change_informative", "Would you like to restart now?"))
|
||||||
|
restart_button = msg_box.addButton(self._tr("restart_now_button", "Restart Now"), QMessageBox.ApplyRole)
|
||||||
|
ok_button = msg_box.addButton(self._tr("ok_button", "OK"), QMessageBox.AcceptRole)
|
||||||
|
msg_box.setDefaultButton(ok_button)
|
||||||
|
msg_box.exec_()
|
||||||
|
|
||||||
|
if msg_box.clickedButton() == restart_button:
|
||||||
|
self.parent_app._request_restart_application()
|
||||||
|
|
||||||
def _save_cookie_and_path(self):
|
def _save_cookie_and_path(self):
|
||||||
|
"""Saves the current download path and/or cookie settings from the main window."""
|
||||||
path_saved = False
|
path_saved = False
|
||||||
cookie_saved = False
|
cookie_saved = False
|
||||||
|
|
||||||
if hasattr(self.parent_app, 'dir_input') and self.parent_app.dir_input:
|
if hasattr(self.parent_app, 'dir_input') and self.parent_app.dir_input:
|
||||||
current_path = self.parent_app.dir_input.text().strip()
|
current_path = self.parent_app.dir_input.text().strip()
|
||||||
if current_path and os.path.isdir(current_path):
|
if current_path and os.path.isdir(current_path):
|
||||||
self.parent_app.settings.setValue(DOWNLOAD_LOCATION_KEY, current_path)
|
self.parent_app.settings.setValue(DOWNLOAD_LOCATION_KEY, current_path)
|
||||||
path_saved = True
|
path_saved = True
|
||||||
|
|
||||||
if hasattr(self.parent_app, 'use_cookie_checkbox'):
|
if hasattr(self.parent_app, 'use_cookie_checkbox'):
|
||||||
use_cookie = self.parent_app.use_cookie_checkbox.isChecked()
|
use_cookie = self.parent_app.use_cookie_checkbox.isChecked()
|
||||||
cookie_content = self.parent_app.cookie_text_input.text().strip()
|
cookie_content = self.parent_app.cookie_text_input.text().strip()
|
||||||
|
|
||||||
if use_cookie and cookie_content:
|
if use_cookie and cookie_content:
|
||||||
self.parent_app.settings.setValue(USE_COOKIE_KEY, True)
|
self.parent_app.settings.setValue(USE_COOKIE_KEY, True)
|
||||||
self.parent_app.settings.setValue(COOKIE_TEXT_KEY, cookie_content)
|
self.parent_app.settings.setValue(COOKIE_TEXT_KEY, cookie_content)
|
||||||
cookie_saved = True
|
cookie_saved = True
|
||||||
else:
|
else:
|
||||||
self.parent_app.settings.setValue(USE_COOKIE_KEY, False)
|
self.parent_app.settings.setValue(USE_COOKIE_KEY, False)
|
||||||
self.parent_app.settings.setValue(COOKIE_TEXT_KEY, "")
|
self.parent_app.settings.setValue(COOKIE_TEXT_KEY, "")
|
||||||
|
|
||||||
self.parent_app.settings.sync()
|
self.parent_app.settings.sync()
|
||||||
if path_saved or cookie_saved:
|
|
||||||
QMessageBox.information(self, "Settings Saved", "Settings have been saved.")
|
# --- User Feedback ---
|
||||||
|
if path_saved and cookie_saved:
|
||||||
|
message = self._tr("settings_save_both_success", "Download location and cookie settings saved.")
|
||||||
|
elif path_saved:
|
||||||
|
message = self._tr("settings_save_path_only_success", "Download location saved. No cookie settings were active to save.")
|
||||||
|
elif cookie_saved:
|
||||||
|
message = self._tr("settings_save_cookie_only_success", "Cookie settings saved. Download location was not set.")
|
||||||
else:
|
else:
|
||||||
QMessageBox.warning(self, "Nothing to Save", "No valid settings to save.")
|
QMessageBox.warning(self, self._tr("settings_save_nothing_title", "Nothing to Save"),
|
||||||
|
self._tr("settings_save_nothing_message", "The download location is not a valid directory and no cookie was active."))
|
||||||
|
return
|
||||||
|
|
||||||
|
QMessageBox.information(self, self._tr("settings_save_success_title", "Settings Saved"), message)
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -138,54 +138,42 @@ def prepare_cookies_for_request(use_cookie_flag, cookie_text_input, selected_coo
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
# In src/utils/network_utils.py
|
|
||||||
|
|
||||||
def extract_post_info(url_string):
|
def extract_post_info(url_string):
|
||||||
"""
|
"""
|
||||||
Parses a URL string to extract the service, user ID, and post ID.
|
Parses a URL string to extract the service, user ID, and post ID.
|
||||||
UPDATED to support Discord, Bunkr, and nhentai URLs.
|
UPDATED to support Discord server/channel URLs.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
url_string (str): The URL to parse.
|
url_string (str): The URL to parse.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
tuple: A tuple containing (service, id1, id2).
|
tuple: A tuple containing (service, id1, id2).
|
||||||
For posts: (service, user_id, post_id).
|
For posts: (service, user_id, post_id).
|
||||||
For Discord: ('discord', server_id, channel_id).
|
For Discord: ('discord', server_id, channel_id).
|
||||||
For Bunkr: ('bunkr', full_url, None).
|
|
||||||
For nhentai: ('nhentai', gallery_id, None).
|
|
||||||
"""
|
"""
|
||||||
if not isinstance(url_string, str) or not url_string.strip():
|
if not isinstance(url_string, str) or not url_string.strip():
|
||||||
return None, None, None
|
return None, None, None
|
||||||
|
|
||||||
stripped_url = url_string.strip()
|
|
||||||
|
|
||||||
# --- Bunkr Check ---
|
|
||||||
bunkr_pattern = re.compile(
|
|
||||||
r"(?:https?://)?(?:[a-zA-Z0-9-]+\.)?bunkr\.(?:si|la|ws|red|black|media|site|is|to|ac|cr|ci|fi|pk|ps|sk|ph|su|ru)|bunkrr\.ru"
|
|
||||||
)
|
|
||||||
if bunkr_pattern.search(stripped_url):
|
|
||||||
return 'bunkr', stripped_url, None
|
|
||||||
|
|
||||||
# --- nhentai Check ---
|
|
||||||
nhentai_match = re.search(r'nhentai\.net/g/(\d+)', stripped_url)
|
|
||||||
if nhentai_match:
|
|
||||||
return 'nhentai', nhentai_match.group(1), None
|
|
||||||
|
|
||||||
# --- Kemono/Coomer/Discord Parsing ---
|
|
||||||
try:
|
try:
|
||||||
parsed_url = urlparse(stripped_url)
|
parsed_url = urlparse(url_string.strip())
|
||||||
path_parts = [part for part in parsed_url.path.strip('/').split('/') if part]
|
path_parts = [part for part in parsed_url.path.strip('/').split('/') if part]
|
||||||
|
|
||||||
|
# Check for new Discord URL format first
|
||||||
|
# e.g., /discord/server/891670433978531850/1252332668805189723
|
||||||
if len(path_parts) >= 3 and path_parts[0].lower() == 'discord' and path_parts[1].lower() == 'server':
|
if len(path_parts) >= 3 and path_parts[0].lower() == 'discord' and path_parts[1].lower() == 'server':
|
||||||
return 'discord', path_parts[2], path_parts[3] if len(path_parts) >= 4 else None
|
service = 'discord'
|
||||||
|
server_id = path_parts[2]
|
||||||
|
channel_id = path_parts[3] if len(path_parts) >= 4 else None
|
||||||
|
return service, server_id, channel_id
|
||||||
|
|
||||||
|
# Standard creator/post format: /<service>/user/<user_id>/post/<post_id>
|
||||||
if len(path_parts) >= 3 and path_parts[1].lower() == 'user':
|
if len(path_parts) >= 3 and path_parts[1].lower() == 'user':
|
||||||
service = path_parts[0]
|
service = path_parts[0]
|
||||||
user_id = path_parts[2]
|
user_id = path_parts[2]
|
||||||
post_id = path_parts[4] if len(path_parts) >= 5 and path_parts[3].lower() == 'post' else None
|
post_id = path_parts[4] if len(path_parts) >= 5 and path_parts[3].lower() == 'post' else None
|
||||||
return service, user_id, post_id
|
return service, user_id, post_id
|
||||||
|
|
||||||
|
# API format: /api/v1/<service>/user/<user_id>...
|
||||||
if len(path_parts) >= 5 and path_parts[0:2] == ['api', 'v1'] and path_parts[3].lower() == 'user':
|
if len(path_parts) >= 5 and path_parts[0:2] == ['api', 'v1'] and path_parts[3].lower() == 'user':
|
||||||
service = path_parts[2]
|
service = path_parts[2]
|
||||||
user_id = path_parts[4]
|
user_id = path_parts[4]
|
||||||
@@ -196,7 +184,7 @@ def extract_post_info(url_string):
|
|||||||
print(f"Debug: Exception during URL parsing for '{url_string}': {e}")
|
print(f"Debug: Exception during URL parsing for '{url_string}': {e}")
|
||||||
|
|
||||||
return None, None, None
|
return None, None, None
|
||||||
|
|
||||||
def get_link_platform(url):
|
def get_link_platform(url):
|
||||||
"""
|
"""
|
||||||
Identifies the platform of a given URL based on its domain.
|
Identifies the platform of a given URL based on its domain.
|
||||||
|
|||||||
Reference in New Issue
Block a user