From cd29f79fd049a2df1431b4d65e59ae6e82cdcd3f Mon Sep 17 00:00:00 2001
From: Paul Wise <pabs3@bonedaddy.net>
Date: Mon, 3 May 2021 16:48:49 +0800
Subject: [PATCH] Switch to the JSON output format for easier parsing

---
 lib/wayback_machine_downloader.rb             | 16 ++++++----------
 lib/wayback_machine_downloader/archive_api.rb | 13 +++++++++++--
 2 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/lib/wayback_machine_downloader.rb b/lib/wayback_machine_downloader.rb
index 04005c8..d1098e0 100644
--- a/lib/wayback_machine_downloader.rb
+++ b/lib/wayback_machine_downloader.rb
@@ -84,7 +84,7 @@ class WaybackMachineDownloader
     # Note: Passing a page index parameter allow us to get more snapshots,
     # but from a less fresh index
     print "Getting snapshot pages"
-    snapshot_list_to_consider = ""
+    snapshot_list_to_consider = []
     snapshot_list_to_consider += get_raw_list_from_api(@base_url, nil)
     print "."
     unless @exact_url
@@ -95,17 +95,15 @@ class WaybackMachineDownloader
         print "."
       end
     end
-    puts " found #{snapshot_list_to_consider.lines.count} snaphots to consider."
+    puts " found #{snapshot_list_to_consider.length} snaphots to consider."
     puts
     snapshot_list_to_consider
   end
 
   def get_file_list_curated
     file_list_curated = Hash.new
-    get_all_snapshots_to_consider.each_line do |line|
-      next unless line.include?('/')
-      file_timestamp = line[0..13].to_i
-      file_url = line[15..-2]
+    get_all_snapshots_to_consider.each do |file_timestamp, file_url|
+      next unless file_url.include?('/')
       file_id = file_url.split('/')[3..-1].join('/')
       file_id = CGI::unescape file_id 
       file_id = file_id.tidy_bytes unless file_id == ""
@@ -130,10 +128,8 @@ class WaybackMachineDownloader
 
   def get_file_list_all_timestamps
     file_list_curated = Hash.new
-    get_all_snapshots_to_consider.each_line do |line|
-      next unless line.include?('/')
-      file_timestamp = line[0..13].to_i
-      file_url = line[15..-2]
+    get_all_snapshots_to_consider.each do |file_timestamp, file_url|
+      next unless file_url.include?('/')
       file_id = file_url.split('/')[3..-1].join('/')
       file_id_and_timestamp = [file_timestamp, file_id].join('/')
       file_id_and_timestamp = CGI::unescape file_id_and_timestamp 
diff --git a/lib/wayback_machine_downloader/archive_api.rb b/lib/wayback_machine_downloader/archive_api.rb
index ef8d3a0..e4b3529 100644
--- a/lib/wayback_machine_downloader/archive_api.rb
+++ b/lib/wayback_machine_downloader/archive_api.rb
@@ -1,14 +1,23 @@
+require 'json'
 require 'uri'
 
 module ArchiveAPI
 
   def get_raw_list_from_api url, page_index
     request_url = URI("https://web.archive.org/cdx/search/xd")
-    params = [["url", url]]
+    params = [["output", "json"], ["url", url]]
     params += parameters_for_api page_index
     request_url.query = URI.encode_www_form(params)
 
-    URI.open(request_url).read
+    begin
+      json = JSON.parse(URI.open(request_url).read)
+      if (json[0] <=> ["timestamp","original"]) == 0
+        json.shift
+      end
+      json
+    rescue JSON::ParserError
+      []
+    end
   end
 
   def parameters_for_api page_index