External/wayback-machine-downloader
1
0
Fork 0
mirror of https://github.com/StrawberryMaster/wayback-machine-downloader.git synced 2025-12-17 17:56:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Switch to the JSON output format for easier parsing

Browse Source
This commit is contained in:
Paul Wise 2021-05-03 16:48:49 +08:00
parent afab72c894
commit cd29f79fd0
No known key found for this signature in database
GPG Key ID: 3116BA5E9FFA69A3
2 changed files with 17 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
lib/wayback_machine_downloader.rb
View File

@ -84,7 +84,7 @@ class WaybackMachineDownloader
# Note: Passing a page index parameter allow us to get more snapshots, # Note: Passing a page index parameter allow us to get more snapshots,
# but from a less fresh index # but from a less fresh index
print "Getting snapshot pages" print "Getting snapshot pages"
snapshot_list_to_consider = "" snapshot_list_to_consider = []
snapshot_list_to_consider += get_raw_list_from_api(@base_url, nil) snapshot_list_to_consider += get_raw_list_from_api(@base_url, nil)
print "." print "."
unless @exact_url unless @exact_url
@ -95,17 +95,15 @@ class WaybackMachineDownloader
print "." print "."
end end
end end
puts " found #{snapshot_list_to_consider.lines.count} snaphots to consider." puts " found #{snapshot_list_to_consider.length} snaphots to consider."
puts puts
snapshot_list_to_consider snapshot_list_to_consider
end end
def get_file_list_curated def get_file_list_curated
file_list_curated = Hash.new file_list_curated = Hash.new
get_all_snapshots_to_consider.each_line do |line| get_all_snapshots_to_consider.each do |file_timestamp, file_url|
next unless line.include?('/') next unless file_url.include?('/')
file_timestamp = line[0..13].to_i
file_url = line[15..-2]
file_id = file_url.split('/')[3..-1].join('/') file_id = file_url.split('/')[3..-1].join('/')
file_id = CGI::unescape file_id file_id = CGI::unescape file_id
file_id = file_id.tidy_bytes unless file_id == "" file_id = file_id.tidy_bytes unless file_id == ""
@ -130,10 +128,8 @@ class WaybackMachineDownloader
def get_file_list_all_timestamps def get_file_list_all_timestamps
file_list_curated = Hash.new file_list_curated = Hash.new
get_all_snapshots_to_consider.each_line do |line| get_all_snapshots_to_consider.each do |file_timestamp, file_url|
next unless line.include?('/') next unless file_url.include?('/')
file_timestamp = line[0..13].to_i
file_url = line[15..-2]
file_id = file_url.split('/')[3..-1].join('/') file_id = file_url.split('/')[3..-1].join('/')
file_id_and_timestamp = [file_timestamp, file_id].join('/') file_id_and_timestamp = [file_timestamp, file_id].join('/')
file_id_and_timestamp = CGI::unescape file_id_and_timestamp file_id_and_timestamp = CGI::unescape file_id_and_timestamp

13
lib/wayback_machine_downloader/archive_api.rb
View File

@ -1,14 +1,23 @@
require 'json'
require 'uri' require 'uri'
module ArchiveAPI module ArchiveAPI
def get_raw_list_from_api url, page_index def get_raw_list_from_api url, page_index
request_url = URI("https://web.archive.org/cdx/search/xd") request_url = URI("https://web.archive.org/cdx/search/xd")
params = [["url", url]] params = [["output", "json"], ["url", url]]
params += parameters_for_api page_index params += parameters_for_api page_index
request_url.query = URI.encode_www_form(params) request_url.query = URI.encode_www_form(params)
URI.open(request_url).read begin
json = JSON.parse(URI.open(request_url).read)
if (json[0] <=> ["timestamp","original"]) == 0
json.shift
end
json
rescue JSON::ParserError
[]
end
end end
def parameters_for_api page_index def parameters_for_api page_index