From 7414c34d0547c4563da43eaca10f0c32c0f4096f Mon Sep 17 00:00:00 2001 From: hartator Date: Sat, 24 Sep 2016 10:21:17 -0700 Subject: [PATCH] Add an additional request to archive.org API to get fresher snapshots --- lib/wayback_machine_downloader.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/wayback_machine_downloader.rb b/lib/wayback_machine_downloader.rb index 7b14043..96d7ebc 100644 --- a/lib/wayback_machine_downloader.rb +++ b/lib/wayback_machine_downloader.rb @@ -77,10 +77,13 @@ class WaybackMachineDownloader end def get_all_snapshots_to_consider + # Note: Passing a page index parameter allow us to get more snapshot, but from a less fresh index print "Getting snapshot pages" snapshot_list_to_consider = "" snapshot_list_to_consider += get_raw_list_from_api(@base_url, nil) print "." + snapshot_list_to_consider += get_raw_list_from_api(@base_url + '/*', nil) + print "." @maximum_pages.times do |page_index| snapshot_list = get_raw_list_from_api(@base_url + '/*', page_index) break if snapshot_list.empty?