Malin/Kemono-Downloader
1
0
Fork 0
mirror of https://github.com/Yuvi9587/Kemono-Downloader.git synced 2025-12-29 16:14:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update downloader_utils.py

Browse Source
This commit is contained in:
Yuvi9587
2025-06-09 16:22:27 +01:00
parent d5d6fd91ef
commit 64a314713e
1 changed files with 48 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
downloader_utils.py
View File

@@ -11,6 +11,7 @@ from concurrent .futures import ThreadPoolExecutor ,Future ,CancelledError ,as_c
import html import html
from PyQt5 .QtCore import QObject ,pyqtSignal ,QThread ,QMutex ,QMutexLocker from PyQt5 .QtCore import QObject ,pyqtSignal ,QThread ,QMutex ,QMutexLocker
from urllib .parse import urlparse from urllib .parse import urlparse
import uuid
try : try :
from mega import Mega from mega import Mega
@@ -1016,6 +1017,14 @@ class PostProcessorWorker :
except OSError as e : except OSError as e :
self .logger (f" ❌ Critical error creating directory '{target_folder_path }': {e }. Skipping file '{api_original_filename }'.") self .logger (f" ❌ Critical error creating directory '{target_folder_path }': {e }. Skipping file '{api_original_filename }'.")
return 0 ,1 ,api_original_filename ,False ,FILE_DOWNLOAD_STATUS_SKIPPED ,None return 0 ,1 ,api_original_filename ,False ,FILE_DOWNLOAD_STATUS_SKIPPED ,None
# Create a unique name stem for the .part file for this specific download operation
# filename_to_save_in_main_path is the intended name before OS-level collision suffixing (_1, _2)
# but multiple concurrent downloads might target the same filename_to_save_in_main_path.
# This is placed here because filename_to_save_in_main_path and api_original_filename are now fully defined.
temp_file_base_for_unique_part, temp_file_ext_for_unique_part = os.path.splitext(filename_to_save_in_main_path if filename_to_save_in_main_path else api_original_filename)
unique_id_for_part_file = uuid.uuid4().hex[:8] # Ensure uuid is imported
unique_part_file_stem_on_disk = f"{temp_file_base_for_unique_part}_{unique_id_for_part_file}"
max_retries =3 max_retries =3
retry_delay =5 retry_delay =5
downloaded_size_bytes =0 downloaded_size_bytes =0
@@ -1026,8 +1035,9 @@ class PostProcessorWorker :
download_successful_flag =False download_successful_flag =False
last_exception_for_retry_later =None last_exception_for_retry_later =None
response_for_this_attempt = None # Initialize outside the loop for finally block
for attempt_num_single_stream in range (max_retries +1 ): for attempt_num_single_stream in range (max_retries +1 ):
response_for_this_attempt = None # Reset for each attempt
if self ._check_pause (f"File download attempt for '{api_original_filename }'"):break if self ._check_pause (f"File download attempt for '{api_original_filename }'"):break
if self .check_cancel ()or (skip_event and skip_event .is_set ()):break if self .check_cancel ()or (skip_event and skip_event .is_set ()):break
try : try :
@@ -1045,11 +1055,17 @@ class PostProcessorWorker :
if self ._check_pause (f"Multipart decision for '{api_original_filename }'"):break if self ._check_pause (f"Multipart decision for '{api_original_filename }'"):break
if attempt_multipart : if attempt_multipart :
response .close () if response_for_this_attempt: # Close the initial response if we're going multipart
self ._emit_signal ('file_download_status',False ) response_for_this_attempt.close()
mp_save_path_base_for_part =os .path .join (target_folder_path ,filename_to_save_in_main_path ) response_for_this_attempt = None # So finally block doesn't try to close it again if multipart fails
# self ._emit_signal ('file_download_status',False ) # Moved to finally
# Use the unique stem for the path passed to download_file_in_parts.
# download_file_in_parts will append ".part" to this.
mp_save_path_for_unique_part_stem_arg = os.path.join(target_folder_path, f"{unique_part_file_stem_on_disk}{temp_file_ext_for_unique_part}")
mp_success ,mp_bytes ,mp_hash ,mp_file_handle =download_file_in_parts ( mp_success ,mp_bytes ,mp_hash ,mp_file_handle =download_file_in_parts (
file_url ,mp_save_path_base_for_part ,total_size_bytes ,num_parts_for_file ,headers ,api_original_filename , file_url ,mp_save_path_for_unique_part_stem_arg ,total_size_bytes ,num_parts_for_file ,headers ,api_original_filename ,
emitter_for_multipart =self .emitter ,cookies_for_chunk_session =cookies_to_use_for_file , emitter_for_multipart =self .emitter ,cookies_for_chunk_session =cookies_to_use_for_file ,
cancellation_event =self .cancellation_event ,skip_event =skip_event ,logger_func =self .logger , cancellation_event =self .cancellation_event ,skip_event =skip_event ,logger_func =self .logger ,
pause_event =self .pause_event pause_event =self .pause_event
@@ -1060,7 +1076,8 @@ class PostProcessorWorker :
calculated_file_hash =mp_hash calculated_file_hash =mp_hash
downloaded_part_file_path =mp_save_path_base_for_part +".part" # This is the actual unique .part file path on disk
downloaded_part_file_path = mp_save_path_for_unique_part_stem_arg + ".part"
was_multipart_download =True was_multipart_download =True
if mp_file_handle :mp_file_handle .close () if mp_file_handle :mp_file_handle .close ()
break break
@@ -1071,13 +1088,15 @@ class PostProcessorWorker :
download_successful_flag =False ;break download_successful_flag =False ;break
else : else :
self .logger (f"⬇️ Downloading (Single Stream): '{api_original_filename }' (Size: {total_size_bytes /(1024 *1024 ):.2f} MB if known) [Base Name: '{filename_to_save_in_main_path }']") self .logger (f"⬇️ Downloading (Single Stream): '{api_original_filename }' (Size: {total_size_bytes /(1024 *1024 ):.2f} MB if known) [Base Name: '{filename_to_save_in_main_path }']")
current_single_stream_part_path =os .path .join (target_folder_path ,filename_to_save_in_main_path +".part") # Use the unique .part filename for single stream download
current_single_stream_part_path = os.path.join(target_folder_path, f"{unique_part_file_stem_on_disk}{temp_file_ext_for_unique_part}.part")
current_attempt_downloaded_bytes =0 current_attempt_downloaded_bytes =0
md5_hasher =hashlib .md5 () md5_hasher =hashlib .md5 ()
last_progress_time =time .time () last_progress_time =time .time ()
single_stream_exception = None
try : try :
with open (current_single_stream_part_path ,'wb')as f_part : with open (current_single_stream_part_path ,'wb') as f_part :
for chunk in response .iter_content (chunk_size =1 *1024 *1024 ): for chunk in response .iter_content (chunk_size =1 *1024 *1024 ):
if self ._check_pause (f"Chunk download for '{api_original_filename }'"):break if self ._check_pause (f"Chunk download for '{api_original_filename }'"):break
if self .check_cancel ()or (skip_event and skip_event .is_set ()):break if self .check_cancel ()or (skip_event and skip_event .is_set ()):break
@@ -1120,13 +1139,18 @@ class PostProcessorWorker :
download_successful_flag = True # Mark THE ENTIRE DOWNLOAD as successful download_successful_flag = True # Mark THE ENTIRE DOWNLOAD as successful
break # Break from the RETRY loop (attempt_num_single_stream) break # Break from the RETRY loop (attempt_num_single_stream)
else: # This attempt was not successful (e.g., incomplete or 0 bytes when not expected) else: # This attempt was not successful (e.g., incomplete or 0 bytes when not expected)
if os .path .exists (current_single_stream_part_path ):os .remove (current_single_stream_part_path ) if os.path.exists(current_single_stream_part_path):
try: os.remove(current_single_stream_part_path)
except OSError as e_rem_part: self.logger(f" -> Failed to remove .part file after failed single stream attempt: {e_rem_part}")
# Let the retry loop continue if more attempts are left; download_successful_flag remains False for this attempt. # Let the retry loop continue if more attempts are left; download_successful_flag remains False for this attempt.
except Exception as e_write : except Exception as e_write :
self .logger (f" ❌ Error writing single-stream to disk for '{api_original_filename }': {e_write }") self .logger (f" ❌ Error writing single-stream to disk for '{api_original_filename }': {e_write }")
if os .path .exists (current_single_stream_part_path ):os .remove (current_single_stream_part_path ) if os .path .exists (current_single_stream_part_path ):os .remove (current_single_stream_part_path )
raise raise
single_stream_exception = e_write
if single_stream_exception:
raise single_stream_exception
except (requests .exceptions .ConnectionError ,requests .exceptions .Timeout ,http .client .IncompleteRead )as e : except (requests .exceptions .ConnectionError ,requests .exceptions .Timeout ,http .client .IncompleteRead )as e :
self .logger (f" ❌ Download Error (Retryable): {api_original_filename }. Error: {e }") self .logger (f" ❌ Download Error (Retryable): {api_original_filename }. Error: {e }")
@@ -1145,7 +1169,9 @@ class PostProcessorWorker :
last_exception_for_retry_later =e last_exception_for_retry_later =e
break break
finally : finally :
self ._emit_signal ('file_download_status',False ) if response_for_this_attempt:
response_for_this_attempt.close()
self ._emit_signal ('file_download_status', False) # Signal that this attempt's file operation is done
final_total_for_progress =total_size_bytes if download_successful_flag and total_size_bytes >0 else downloaded_size_bytes final_total_for_progress =total_size_bytes if download_successful_flag and total_size_bytes >0 else downloaded_size_bytes
self ._emit_signal ('file_progress',api_original_filename ,(downloaded_size_bytes ,final_total_for_progress )) self ._emit_signal ('file_progress',api_original_filename ,(downloaded_size_bytes ,final_total_for_progress ))
@@ -1263,19 +1289,20 @@ class PostProcessorWorker :
final_filename_on_disk =filename_after_compression final_filename_on_disk =filename_after_compression
if not (self .manga_mode_active and self .manga_filename_style ==STYLE_DATE_BASED ): # Always apply suffixing if a file with the exact name already exists in the target folder.
temp_base ,temp_ext =os .path .splitext (final_filename_on_disk ) temp_base ,temp_ext =os .path .splitext (final_filename_on_disk )
suffix_counter =1 suffix_counter =1
while os .path .exists (os .path .join (effective_save_folder ,final_filename_on_disk )): while os .path .exists (os .path .join (effective_save_folder ,final_filename_on_disk )):
final_filename_on_disk =f"{temp_base }_{suffix_counter }{temp_ext }" final_filename_on_disk =f"{temp_base }_{suffix_counter }{temp_ext }"
suffix_counter +=1 suffix_counter +=1
if final_filename_on_disk !=filename_after_compression : if final_filename_on_disk !=filename_after_compression : # Log if a suffix was actually applied
self .logger (f" Applied numeric suffix in '{os .path .basename (effective_save_folder )}': '{final_filename_on_disk }' (was '{filename_after_compression }')") self .logger (f" Applied numeric suffix in '{os .path .basename (effective_save_folder )}': '{final_filename_on_disk }' (was '{filename_after_compression }')")
if self ._check_pause (f"File saving for '{final_filename_on_disk }'"):return 0 ,1 ,final_filename_on_disk ,was_original_name_kept_flag ,FILE_DOWNLOAD_STATUS_SKIPPED ,None if self ._check_pause (f"File saving for '{final_filename_on_disk }'"):return 0 ,1 ,final_filename_on_disk ,was_original_name_kept_flag ,FILE_DOWNLOAD_STATUS_SKIPPED ,None
final_save_path =os .path .join (effective_save_folder ,final_filename_on_disk ) final_save_path =os .path .join (effective_save_folder ,final_filename_on_disk )
try : try :
if data_to_write_io : if data_to_write_io :
with open (final_save_path ,'wb')as f_out : with open (final_save_path ,'wb')as f_out :
time.sleep(0.05) # Small delay before writing compressed data
f_out .write (data_to_write_io .getvalue ()) f_out .write (data_to_write_io .getvalue ())
if downloaded_part_file_path and os .path .exists (downloaded_part_file_path ): if downloaded_part_file_path and os .path .exists (downloaded_part_file_path ):
@@ -1285,6 +1312,7 @@ class PostProcessorWorker :
self .logger (f" -> Failed to remove .part after compression: {e_rem }") self .logger (f" -> Failed to remove .part after compression: {e_rem }")
else : else :
if downloaded_part_file_path and os .path .exists (downloaded_part_file_path ): if downloaded_part_file_path and os .path .exists (downloaded_part_file_path ):
time.sleep(0.1) # Small delay before rename for non-compressed files
os .rename (downloaded_part_file_path ,final_save_path ) os .rename (downloaded_part_file_path ,final_save_path )
else : else :
raise FileNotFoundError (f"Original .part file not found for saving: {downloaded_part_file_path }") raise FileNotFoundError (f"Original .part file not found for saving: {downloaded_part_file_path }")
@@ -1295,7 +1323,7 @@ class PostProcessorWorker :
time .sleep (0.05 ) time .sleep (0.05 )
return 1 ,0 ,final_filename_saved_for_return ,was_original_name_kept_flag ,FILE_DOWNLOAD_STATUS_SUCCESS ,None return 1 ,0 ,final_filename_saved_for_return ,was_original_name_kept_flag ,FILE_DOWNLOAD_STATUS_SUCCESS ,None
except Exception as save_err : except Exception as save_err :
self .logger (f"❌ Save Fail for '{final_filename_on_disk }': {save_err }") self .logger (f"->>Save Fail for '{final_filename_on_disk }': {save_err }")
if os .path .exists (final_save_path ): if os .path .exists (final_save_path ):
try :os .remove (final_save_path ); try :os .remove (final_save_path );
except OSError :self .logger (f" -> Failed to remove partially saved file: {final_save_path }") except OSError :self .logger (f" -> Failed to remove partially saved file: {final_save_path }")