4 Commits

Author SHA1 Message Date
Felipe
4160ff5e4a Bumping version 2025-06-18 18:05:31 +00:00
Felipe
2490109cfe Merge pull request #17 from elidickinson/fix-exact-url
don’t append /* when using —exact-url
2025-06-15 22:18:40 -03:00
Eli Dickinson
c3c5b8446a don’t append /* when —exact-url 2025-06-15 13:26:11 -04:00
Felipe
18357a77ed Correct file path and sanitization in Windows
Not only we weren't normalizing the file directories, we were also agressively sanitizing incorrect characters, leading to some funny stuff on Windows. Fixes #16
2025-06-15 13:48:11 +00:00
3 changed files with 39 additions and 22 deletions

View File

@@ -113,7 +113,7 @@ class WaybackMachineDownloader
include ArchiveAPI
VERSION = "2.3.8"
VERSION = "2.3.9"
DEFAULT_TIMEOUT = 30
MAX_RETRIES = 3
RETRY_DELAY = 2
@@ -131,7 +131,11 @@ class WaybackMachineDownloader
validate_params(params)
@base_url = params[:base_url]
@exact_url = params[:exact_url]
@directory = params[:directory]
if params[:directory]
@directory = File.expand_path(params[:directory])
else
@directory = nil
end
@all_timestamps = params[:all_timestamps]
@from_timestamp = params[:from_timestamp].to_i
@to_timestamp = params[:to_timestamp].to_i
@@ -165,13 +169,11 @@ class WaybackMachineDownloader
def backup_path
if @directory
if @directory[-1] == '/'
@directory
else
@directory + '/'
end
# because @directory is already an absolute path, we just ensure it exists
@directory
else
'websites/' + backup_name + '/'
# ensure the default path is absolute and normalized
File.expand_path(File.join('websites', backup_name))
end
end
@@ -638,21 +640,35 @@ class WaybackMachineDownloader
file_url = file_remote_info[:file_url].encode(current_encoding)
file_id = file_remote_info[:file_id]
file_timestamp = file_remote_info[:timestamp]
file_path_elements = file_id.split('/')
# sanitize file_id to ensure it is a valid path component
raw_path_elements = file_id.split('/')
sanitized_path_elements = raw_path_elements.map do |element|
if Gem.win_platform?
# for Windows, we need to sanitize path components to avoid invalid characters
# this prevents issues with file names that contain characters not allowed in
# Windows file systems. See # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions
element.gsub(/[:\*?"<>\|\&\=\/\\]/, ->(match) { '%' + match.ord.to_s(16).upcase })
else
element
end
end
current_backup_path = backup_path
if file_id == ""
dir_path = backup_path
file_path = backup_path + 'index.html'
elsif file_url[-1] == '/' or not file_path_elements[-1].include? '.'
dir_path = backup_path + file_path_elements[0..-1].join('/')
file_path = backup_path + file_path_elements[0..-1].join('/') + '/index.html'
dir_path = current_backup_path
file_path = File.join(dir_path, 'index.html')
elsif file_url[-1] == '/' || (sanitized_path_elements.last && !sanitized_path_elements.last.include?('.'))
# if file_id is a directory, we treat it as such
dir_path = File.join(current_backup_path, *sanitized_path_elements)
file_path = File.join(dir_path, 'index.html')
else
dir_path = backup_path + file_path_elements[0..-2].join('/')
file_path = backup_path + file_path_elements[0..-1].join('/')
end
if Gem.win_platform?
dir_path = dir_path.gsub(/[:*?&=<>\\|]/) {|s| '%' + s.ord.to_s(16) }
file_path = file_path.gsub(/[:*?&=<>\\|]/) {|s| '%' + s.ord.to_s(16) }
# if file_id is a file, we treat it as such
filename = sanitized_path_elements.pop
dir_path = File.join(current_backup_path, *sanitized_path_elements)
file_path = File.join(dir_path, filename)
end
# check existence *before* download attempt

View File

@@ -7,7 +7,8 @@ module ArchiveAPI
# Automatically append /* if the URL doesn't contain a path after the domain
# This is a workaround for an issue with the API and *some* domains.
# See https://github.com/StrawberryMaster/wayback-machine-downloader/issues/6
if url && !url.match(/^https?:\/\/.*\//i)
# But don't do this when exact_url flag is set
if url && !url.match(/^https?:\/\/.*\//i) && !@exact_url
url = "#{url}/*"
end

View File

@@ -1,6 +1,6 @@
Gem::Specification.new do |s|
s.name = "wayback_machine_downloader_straw"
s.version = "2.3.8"
s.version = "2.3.9"
s.executables << "wayback_machine_downloader"
s.summary = "Download an entire website from the Wayback Machine."
s.description = "Download complete websites from the Internet Archive's Wayback Machine. While the Wayback Machine (archive.org) excellently preserves web history, it lacks a built-in export functionality; this gem does just that, allowing you to download entire archived websites. (This is a significant rewrite of the original wayback_machine_downloader gem by hartator, with enhanced features and performance improvements.)"