External/wayback-machine-downloader
1
0
Fork 0
mirror of https://github.com/StrawberryMaster/wayback-machine-downloader.git synced 2025-12-29 16:16:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Switch to the JSON output format for easier parsing

Browse Source
This commit is contained in:
Paul Wise
2021-05-03 16:48:49 +08:00
parent afab72c894
commit cd29f79fd0
2 changed files with 17 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
lib/wayback_machine_downloader.rb
View File

@@ -84,7 +84,7 @@ class WaybackMachineDownloader
# Note: Passing a page index parameter allow us to get more snapshots,
# but from a less fresh index
print "Getting snapshot pages"
snapshot_list_to_consider = ""
snapshot_list_to_consider = []
snapshot_list_to_consider += get_raw_list_from_api(@base_url, nil)
print "."
unless @exact_url
@@ -95,17 +95,15 @@ class WaybackMachineDownloader
print "."
end
end
puts " found #{snapshot_list_to_consider.lines.count} snaphots to consider."
puts " found #{snapshot_list_to_consider.length} snaphots to consider."
puts
snapshot_list_to_consider
end
def get_file_list_curated
file_list_curated = Hash.new
get_all_snapshots_to_consider.each_line do |line|
next unless line.include?('/')
file_timestamp = line[0..13].to_i
file_url = line[15..-2]
get_all_snapshots_to_consider.each do |file_timestamp, file_url|
next unless file_url.include?('/')
file_id = file_url.split('/')[3..-1].join('/')
file_id = CGI::unescape file_id
file_id = file_id.tidy_bytes unless file_id == ""
@@ -130,10 +128,8 @@ class WaybackMachineDownloader
def get_file_list_all_timestamps
file_list_curated = Hash.new
get_all_snapshots_to_consider.each_line do |line|
next unless line.include?('/')
file_timestamp = line[0..13].to_i
file_url = line[15..-2]
get_all_snapshots_to_consider.each do |file_timestamp, file_url|
next unless file_url.include?('/')
file_id = file_url.split('/')[3..-1].join('/')
file_id_and_timestamp = [file_timestamp, file_id].join('/')
file_id_and_timestamp = CGI::unescape file_id_and_timestamp