Commit

src/core/bunkr_client.py

@@ -0,0 +1,249 @@
+import logging
+import os
+import re
+import requests
+import html
+import time
+import datetime
+import urllib.parse
+import json
+import random
+import binascii
+import itertools
+
+class MockMessage:
+    Directory = 1
+    Url = 2
+    Version = 3
+
+class AlbumException(Exception): pass
+class ExtractionError(AlbumException): pass
+class HttpError(ExtractionError):
+    def __init__(self, message="", response=None):
+        self.response = response
+        self.status = response.status_code if response is not None else 0
+        super().__init__(message)
+class ControlException(AlbumException): pass
+class AbortExtraction(ExtractionError, ControlException): pass
+
+try:
+    re_compile = re._compiler.compile
+except AttributeError:
+    re_compile = re.sre_compile.compile
+HTML_RE = re_compile(r"<[^>]+>")
+def extr(txt, begin, end, default=""):
+    try:
+        first = txt.index(begin) + len(begin)
+        return txt[first:txt.index(end, first)]
+    except Exception: return default
+def extract_iter(txt, begin, end, pos=None):
+    try:
+        index = txt.index
+        lbeg = len(begin)
+        lend = len(end)
+        while True:
+            first = index(begin, pos) + lbeg
+            last = index(end, first)
+            pos = last + lend
+            yield txt[first:last]
+    except Exception: return
+def split_html(txt):
+    try: return [html.unescape(x).strip() for x in HTML_RE.split(txt) if x and not x.isspace()]
+    except TypeError: return []
+def parse_datetime(date_string, format="%Y-%m-%dT%H:%M:%S%z", utcoffset=0):
+    try:
+        d = datetime.datetime.strptime(date_string, format)
+        o = d.utcoffset()
+        if o is not None: d = d.replace(tzinfo=None, microsecond=0) - o
+        else:
+            if d.microsecond: d = d.replace(microsecond=0)
+            if utcoffset: d += datetime.timedelta(0, utcoffset * -3600)
+        return d
+    except (TypeError, IndexError, KeyError, ValueError, OverflowError): return None
+unquote = urllib.parse.unquote
+unescape = html.unescape
+
+# --- From: util.py ---
+def decrypt_xor(encrypted, key, base64=True, fromhex=False):
+    if base64: encrypted = binascii.a2b_base64(encrypted)
+    if fromhex: encrypted = bytes.fromhex(encrypted.decode())
+    div = len(key)
+    return bytes([encrypted[i] ^ key[i % div] for i in range(len(encrypted))]).decode()
+def advance(iterable, num):
+    iterator = iter(iterable)
+    next(itertools.islice(iterator, num, num), None)
+    return iterator
+def json_loads(s): return json.loads(s)
+def json_dumps(obj): return json.dumps(obj, separators=(",", ":"))
+
+# --- From: common.py ---
+class Extractor:
+    def __init__(self, match, logger):
+        self.log = logger
+        self.url = match.string
+        self.match = match
+        self.groups = match.groups()
+        self.session = requests.Session()
+        self.session.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0"
+    @classmethod
+    def from_url(cls, url, logger):
+        if isinstance(cls.pattern, str): cls.pattern = re.compile(cls.pattern)
+        match = cls.pattern.match(url)
+        return cls(match, logger) if match else None
+    def __iter__(self): return self.items()
+    def items(self): yield MockMessage.Version, 1
+    def request(self, url, method="GET", fatal=True, **kwargs):
+        tries = 1
+        while True:
+            try:
+                response = self.session.request(method, url, **kwargs)
+                if response.status_code < 400: return response
+                msg = f"'{response.status_code} {response.reason}' for '{response.url}'"
+            except requests.exceptions.RequestException as exc:
+                msg = str(exc)
+            
+            self.log.info("%s (retrying...)", msg)
+            if tries > 4: break
+            time.sleep(tries)
+            tries += 1
+        if not fatal: return None
+        raise HttpError(msg)
+    def request_json(self, url, **kwargs):
+        response = self.request(url, **kwargs)
+        try: return json_loads(response.text)
+        except Exception as exc:
+            self.log.warning("%s: %s", exc.__class__.__name__, exc)
+            if not kwargs.get("fatal", True): return {}
+            raise
+
+# --- From: bunkr.py (Adapted) ---
+BASE_PATTERN_BUNKR = r"(?:https?://)?(?:[a-zA-Z0-9-]+\.)?(bunkr\.(?:si|la|ws|red|black|media|site|is|to|ac|cr|ci|fi|pk|ps|sk|ph|su)|bunkrr\.ru)"
+DOMAINS = ["bunkr.si", "bunkr.ws", "bunkr.la", "bunkr.red", "bunkr.black", "bunkr.media", "bunkr.site"]
+CF_DOMAINS = set()
+
+class BunkrAlbumExtractor(Extractor):
+    category = "bunkr"
+    root = "https://bunkr.si"
+    root_dl = "https://get.bunkrr.su"
+    root_api = "https://apidl.bunkr.ru"
+    pattern = re.compile(BASE_PATTERN_BUNKR + r"/a/([^/?#]+)")
+
+    def __init__(self, match, logger):
+        super().__init__(match, logger)
+        domain_match = re.search(BASE_PATTERN_BUNKR, match.string)
+        if domain_match:
+            self.root = "https://" + domain_match.group(1)
+        self.endpoint = self.root_api + "/api/_001_v2"
+        self.album_id = self.groups[-1]
+
+    def items(self):
+        page = self.request(self.url).text
+        title = unescape(unescape(extr(page, 'property="og:title" content="', '"')))
+        items_html = list(extract_iter(page, '<div class="grid-images_box', "</a>"))
+
+        album_data = {
+            "album_id": self.album_id, 
+            "album_name": title,
+            "count": len(items_html),
+        }
+        yield MockMessage.Directory, album_data, {}
+
+        for item_html in items_html:
+            try:
+                webpage_url = unescape(extr(item_html, ' href="', '"'))
+                if webpage_url.startswith("/"):
+                    webpage_url = self.root + webpage_url
+                
+                file_data = self._extract_file(webpage_url)
+                info = split_html(item_html)
+
+                if not file_data.get("name"):
+                    file_data["name"] = info[-3]
+                
+                yield MockMessage.Url, file_data, {}
+            except Exception as exc:
+                self.log.error("%s: %s", exc.__class__.__name__, exc)
+
+    def _extract_file(self, webpage_url):
+        page = self.request(webpage_url).text
+        data_id = extr(page, 'data-file-id="', '"')
+        referer = self.root_dl + "/file/" + data_id
+        headers = {"Referer": referer, "Origin": self.root_dl}
+        data = self.request_json(self.endpoint, method="POST", headers=headers, json={"id": data_id})
+        
+        file_url = decrypt_xor(data["url"], f"SECRET_KEY_{data['timestamp'] // 3600}".encode()) if data.get("encrypted") else data["url"]
+        file_name = extr(page, "<h1", "<").rpartition(">")[2]
+
+        return {
+            "url": file_url, 
+            "name": unescape(file_name), 
+            "_http_headers": {"Referer": referer}
+        }
+
+class BunkrMediaExtractor(BunkrAlbumExtractor):
+    pattern = re.compile(BASE_PATTERN_BUNKR + r"(/[fvid]/[^/?#]+)")
+    def items(self):
+        try:
+            media_path = self.groups[-1]
+            file_data = self._extract_file(self.root + media_path)
+            album_data = {"album_name": file_data.get("name", "bunkr_media"), "count": 1}
+
+            yield MockMessage.Directory, album_data, {}
+            yield MockMessage.Url, file_data, {}
+
+        except Exception as exc:
+            self.log.error("%s: %s", exc.__class__.__name__, exc)
+            yield MockMessage.Directory, {"album_name": "error", "count": 0}, {}
+
+# ==============================================================================
+# --- PUBLIC API FOR THE GUI ---
+# ==============================================================================
+
+def get_bunkr_extractor(url, logger):
+    """Selects the correct Bunkr extractor based on the URL pattern."""
+    if BunkrAlbumExtractor.pattern.match(url):
+        logger.info("Bunkr Album URL detected.")
+        return BunkrAlbumExtractor.from_url(url, logger)
+    elif BunkrMediaExtractor.pattern.match(url):
+        logger.info("Bunkr Media URL detected.")
+        return BunkrMediaExtractor.from_url(url, logger)
+    else:
+        logger.error(f"No suitable Bunkr extractor found for URL: {url}")
+        return None
+
+def fetch_bunkr_data(url, logger):
+    """
+    Main function to be called from the GUI.
+    It extracts all file information from a Bunkr URL.
+    
+    Returns:
+        A tuple of (album_name, list_of_files)
+        - album_name (str): The name of the album.
+        - list_of_files (list): A list of dicts, each containing 'url', 'name', and '_http_headers'.
+        Returns (None, None) on failure.
+    """
+    extractor = get_bunkr_extractor(url, logger)
+    if not extractor:
+        return None, None
+
+    try:
+        album_name = "default_bunkr_album"
+        files_to_download = []
+        for msg_type, data, metadata in extractor:
+            if msg_type == MockMessage.Directory:
+                raw_album_name = data.get('album_name', 'untitled')
+                album_name = re.sub(r'[<>:"/\\|?*]', '_', raw_album_name).strip() or "untitled"
+                logger.info(f"Processing Bunkr album: {album_name}")
+            elif msg_type == MockMessage.Url:
+                # data here is the file_data dictionary
+                files_to_download.append(data)
+        
+        if not files_to_download:
+            logger.warning("No files found to download from the Bunkr URL.")
+            return None, None
+            
+        return album_name, files_to_download
+    
+    except Exception as e:
+        logger.error(f"An error occurred while extracting Bunkr info: {e}", exc_info=True)
+        return None, None

src/core/erome_client.py

@@ -0,0 +1,147 @@
+# src/core/erome_client.py
+
+import os
+import re
+import html
+import time
+import urllib.parse
+import requests
+from datetime import datetime
+
+# #############################################################################
+# SECTION: Utility functions adapted from the original script
+# #############################################################################
+
+def extr(txt, begin, end, default=""):
+    """Stripped-down version of 'extract()' to find text between two delimiters."""
+    try:
+        first = txt.index(begin) + len(begin)
+        return txt[first:txt.index(end, first)]
+    except (ValueError, IndexError):
+        return default
+
+def extract_iter(txt, begin, end):
+    """Yields all occurrences of text between two delimiters."""
+    try:
+        index = txt.index
+        lbeg = len(begin)
+        lend = len(end)
+        pos = 0
+        while True:
+            first = index(begin, pos) + lbeg
+            last = index(end, first)
+            pos = last + lend
+            yield txt[first:last]
+    except (ValueError, IndexError):
+        return
+
+def nameext_from_url(url):
+    """Extracts filename and extension from a URL."""
+    data = {}
+    filename = urllib.parse.unquote(url.partition("?")[0].rpartition("/")[2])
+    name, _, ext = filename.rpartition(".")
+    if name and len(ext) <= 16:
+        data["filename"], data["extension"] = name, ext.lower()
+    else:
+        data["filename"], data["extension"] = filename, ""
+    return data
+
+def parse_timestamp(ts, default=None):
+    """Creates a datetime object from a Unix timestamp."""
+    try:
+        # Use fromtimestamp for simplicity and compatibility
+        return datetime.fromtimestamp(int(ts))
+    except (ValueError, TypeError):
+        return default
+
+# #############################################################################
+# SECTION: Main Erome Fetching Logic
+# #############################################################################
+
+def fetch_erome_data(url, logger):
+    """
+    Identifies and extracts all media files from an Erome album URL.
+
+    Args:
+        url (str): The Erome album URL (e.g., https://www.erome.com/a/albumID).
+        logger (function): A function to log progress messages.
+
+    Returns:
+        tuple: A tuple containing (album_folder_name, list_of_file_dicts).
+               Returns (None, []) if data extraction fails.
+    """
+    album_id_match = re.search(r"/a/(\w+)", url)
+    if not album_id_match:
+        logger(f"Error: The URL '{url}' does not appear to be a valid Erome album link.")
+        return None, []
+
+    album_id = album_id_match.group(1)
+    page_url = f"https://www.erome.com/a/{album_id}"
+    
+    session = requests.Session()
+    session.headers.update({
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
+        "Referer": "https://www.erome.com/"
+    })
+
+    try:
+        logger(f"   Fetching Erome album page: {page_url}")
+        # Add a loop to handle "Please wait" pages
+        for attempt in range(5):
+            response = session.get(page_url, timeout=30)
+            response.raise_for_status()
+            page_content = response.text
+            if "<title>Please wait a few moments</title>" in page_content:
+                logger(f"   Cloudflare check detected. Waiting 5 seconds... (Attempt {attempt + 1}/5)")
+                time.sleep(5)
+                continue
+            break
+        else:
+            logger("   Error: Could not bypass Cloudflare check after several attempts.")
+            return None, []
+
+        title = html.unescape(extr(page_content, 'property="og:title" content="', '"'))
+        user = urllib.parse.unquote(extr(page_content, 'href="https://www.erome.com/', '"', default="unknown_user"))
+        
+        # Sanitize title and user for folder creation
+        sanitized_title = re.sub(r'[<>:"/\\|?*]', '_', title).strip()
+        sanitized_user = re.sub(r'[<>:"/\\|?*]', '_', user).strip()
+        
+        album_folder_name = f"Erome - {sanitized_user} - {sanitized_title} [{album_id}]"
+        
+        urls = []
+        # Split the page content by media groups to find all videos
+        media_groups = page_content.split('<div class="media-group"')
+        for group in media_groups[1:]: # Skip the part before the first media group
+            # Prioritize <source> tag, fall back to data-src for images
+            video_url = extr(group, '<source src="', '"') or extr(group, 'data-src="', '"')
+            if video_url:
+                urls.append(video_url)
+
+        if not urls:
+            logger("   Warning: No media URLs found on the album page.")
+            return album_folder_name, []
+
+        logger(f"   Found {len(urls)} media files in album '{title}'.")
+        
+        file_list = []
+        for i, file_url in enumerate(urls, 1):
+            filename_info = nameext_from_url(file_url)
+            # Create a clean, descriptive filename
+            filename = f"{album_id}_{sanitized_title}_{i:03d}.{filename_info.get('extension', 'mp4')}"
+            
+            file_data = {
+                "url": file_url,
+                "filename": filename,
+                "headers": {"Referer": page_url},
+            }
+            file_list.append(file_data)
+            
+        return album_folder_name, file_list
+
+    except requests.exceptions.RequestException as e:
+        logger(f"   Error fetching Erome page: {e}")
+        return None, []
+    except Exception as e:
+        logger(f"   An unexpected error occurred during Erome extraction: {e}")
+        return None, []

src/core/saint2_client.py

@@ -0,0 +1,173 @@
+# src/core/saint2_client.py
+
+import os
+import re as re_module
+import html
+import urllib.parse
+import requests
+
+# ##############################################################################
+# SECTION: Utility functions adapted from the original script
+# ##############################################################################
+
+PATTERN_CACHE = {}
+
+def re(pattern):
+    """Compile a regular expression pattern and cache it."""
+    try:
+        return PATTERN_CACHE[pattern]
+    except KeyError:
+        p = PATTERN_CACHE[pattern] = re_module.compile(pattern)
+        return p
+
+def extract_from(txt, pos=None, default=""):
+    """Returns a function that extracts text between two delimiters from 'txt'."""
+    def extr(begin, end, index=txt.find, txt=txt):
+        nonlocal pos
+        try:
+            start_pos = pos if pos is not None else 0
+            first = index(begin, start_pos) + len(begin)
+            last = index(end, first)
+            if pos is not None:
+                pos = last + len(end)
+            return txt[first:last]
+        except (ValueError, IndexError):
+            return default
+    return extr
+
+def nameext_from_url(url):
+    """Extract filename and extension from a URL."""
+    data = {}
+    filename = urllib.parse.unquote(url.partition("?")[0].rpartition("/")[2])
+    name, _, ext = filename.rpartition(".")
+    if name and len(ext) <= 16:
+        data["filename"], data["extension"] = name, ext.lower()
+    else:
+        data["filename"], data["extension"] = filename, ""
+    return data
+
+# ##############################################################################
+# SECTION: Extractor Logic adapted for the main application
+# ##############################################################################
+
+class BaseExtractor:
+    """A simplified base class for extractors."""
+    def __init__(self, match, session, logger):
+        self.match = match
+        self.groups = match.groups()
+        self.session = session
+        self.log = logger
+
+    def request(self, url, **kwargs):
+        """Makes an HTTP request using the session."""
+        try:
+            response = self.session.get(url, **kwargs)
+            response.raise_for_status()
+            return response
+        except requests.exceptions.RequestException as e:
+            self.log(f"Error making request to {url}: {e}")
+            return None
+
+class SaintAlbumExtractor(BaseExtractor):
+    """Extractor for saint.su albums."""
+    root = "https://saint2.su"
+    pattern = re(r"(?:https?://)?saint\d*\.(?:su|pk|cr|to)/a/([^/?#]+)")
+
+    def items(self):
+        """Generator that yields all files from an album."""
+        album_id = self.groups[0]
+        response = self.request(f"{self.root}/a/{album_id}")
+        if not response:
+            return None, []
+
+        extr = extract_from(response.text)
+        title = extr("<title>", "<").rpartition(" - ")[0]
+        self.log(f"Downloading album: {title}")
+
+        files_html = re_module.findall(r'<a class="image".*?</a>', response.text, re_module.DOTALL)
+        file_list = []
+        for i, file_html in enumerate(files_html, 1):
+            file_extr = extract_from(file_html)
+            file_url = html.unescape(file_extr("onclick=\"play('", "'"))
+            if not file_url:
+                continue
+
+            filename_info = nameext_from_url(file_url)
+            filename = f"{filename_info['filename']}.{filename_info['extension']}"
+
+            file_data = {
+                "url": file_url,
+                "filename": filename,
+                "headers": {"Referer": response.url},
+            }
+            file_list.append(file_data)
+        
+        return title, file_list
+
+class SaintMediaExtractor(BaseExtractor):
+    """Extractor for single saint.su media links."""
+    root = "https://saint2.su"
+    pattern = re(r"(?:https?://)?saint\d*\.(?:su|pk|cr|to)(/(embe)?d/([^/?#]+))")
+
+    def items(self):
+        """Generator that yields the single file from a media page."""
+        path, embed, media_id = self.groups
+        url = self.root + path
+        response = self.request(url)
+        if not response:
+            return None, []
+
+        extr = extract_from(response.text)
+        file_url = ""
+        title = extr("<title>", "<").rpartition(" - ")[0] or media_id
+
+        if embed:  # /embed/ link
+            file_url = html.unescape(extr('<source src="', '"'))
+        else:  # /d/ link
+            file_url = html.unescape(extr('<a href="', '"'))
+
+        if not file_url:
+            self.log("Could not find video URL on the page.")
+            return title, []
+
+        filename_info = nameext_from_url(file_url)
+        filename = f"{filename_info['filename'] or media_id}.{filename_info['extension'] or 'mp4'}"
+
+        file_data = {
+            "url": file_url,
+            "filename": filename,
+            "headers": {"Referer": response.url}
+        }
+        
+        return title, [file_data]
+
+
+def fetch_saint2_data(url, logger):
+    """
+    Identifies the correct extractor for a saint2.su URL and returns the data.
+    
+    Args:
+        url (str): The saint2.su URL.
+        logger (function): A function to log progress messages.
+        
+    Returns:
+        tuple: A tuple containing (album_title, list_of_file_dicts).
+               Returns (None, []) if no data could be fetched.
+    """
+    extractors = [SaintMediaExtractor, SaintAlbumExtractor]
+    session = requests.Session()
+    session.headers.update({
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+    })
+
+    for extractor_cls in extractors:
+        match = extractor_cls.pattern.match(url)
+        if match:
+            extractor = extractor_cls(match, session, logger)
+            album_title, files = extractor.items()
+            # Sanitize the album title to be a valid folder name
+            sanitized_title = re_module.sub(r'[<>:"/\\|?*]', '_', album_title) if album_title else "saint2_download"
+            return sanitized_title, files
+
+    logger(f"Error: The URL '{url}' does not match a known saint2 pattern.")
+    return None, []