Switch to the JSON output format for easier parsing

lib/wayback_machine_downloader/archive_api.rb

@@ -1,14 +1,23 @@
+require 'json'
 require 'uri'
 
 module ArchiveAPI
 
   def get_raw_list_from_api url, page_index
     request_url = URI("https://web.archive.org/cdx/search/xd")
-    params = [["url", url]]
+    params = [["output", "json"], ["url", url]]
     params += parameters_for_api page_index
     request_url.query = URI.encode_www_form(params)
 
-    URI.open(request_url).read
+    begin
+      json = JSON.parse(URI.open(request_url).read)
+      if (json[0] <=> ["timestamp","original"]) == 0
+        json.shift
+      end
+      json
+    rescue JSON::ParserError
+      []
+    end
   end
 
   def parameters_for_api page_index