diff --git a/lib/wayback_machine_downloader.rb b/lib/wayback_machine_downloader.rb index ac43178..b33dfa5 100644 --- a/lib/wayback_machine_downloader.rb +++ b/lib/wayback_machine_downloader.rb @@ -1,3 +1,5 @@ +# encoding: UTF-8 + require 'open-uri' require 'fileutils' require_relative 'wayback_machine_downloader/tidy_bytes' @@ -27,11 +29,6 @@ class WaybackMachineDownloader file_list_curated = Hash.new [index_file_list_raw, all_file_list_raw].each do |file| file.each_line do |line| - line = line.tidy_bytes - unless line - puts "Malformed line, ignoring." - next - end line = line.split(' ') file_timestamp = line[1].to_i file_url = line[2]