Ignore urls with particularly long file name #4

This commit is contained in:
hartator 2015-08-21 20:43:31 -05:00
parent 919210fe64
commit 47ff4bed13

View File

@ -80,6 +80,7 @@ class WaybackMachineDownloader
end
unless File.exists? file_path
structure_dir_path dir_path
begin
open(file_path, "wb") do |file|
begin
open("http://web.archive.org/web/#{timestamp}id_/#{file_url}") do |uri|
@ -92,6 +93,9 @@ class WaybackMachineDownloader
puts "#{file_url} # #{e}"
end
end
rescue Errno::ENAMETOOLONG => e
puts "#{file_url} # #{e}"
end
puts "#{file_url} -> #{file_path} (#{count}/#{file_list_curated.size})"
else
puts "#{file_url} # #{file_path} already exists. (#{count}/#{file_list_curated.size})"