From 94acdd6a7d649780b1410c4af77dd2a22ad76654 Mon Sep 17 00:00:00 2001 From: hartator Date: Fri, 16 Sep 2016 15:02:55 -0500 Subject: [PATCH] Skip lines with no actionable data --- lib/wayback_machine_downloader.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/wayback_machine_downloader.rb b/lib/wayback_machine_downloader.rb index c8fe3e2..57a2ded 100644 --- a/lib/wayback_machine_downloader.rb +++ b/lib/wayback_machine_downloader.rb @@ -78,6 +78,7 @@ class WaybackMachineDownloader file_list_curated = Hash.new [index_file_list_raw, all_file_list_raw].each do |file| file.each_line do |line| + next if line.size < 20 file_timestamp = line[0..13].to_i file_url = line[15..-2] file_id = file_url.split('/')[3..-1].join('/')