External/wayback-machine-downloader
1
0
Fork 0
mirror of https://github.com/StrawberryMaster/wayback-machine-downloader.git synced 2025-12-18 10:16:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Construct the cdx API query using a URI object

Browse Source 
This avoids problems related to URL encoding.

Obsoletes: https://github.com/hartator/wayback-machine-downloader/pull/116
This commit is contained in:
Paul Wise 2021-05-03 14:01:04 +08:00
parent e6707a9d8b
commit afab72c894
No known key found for this signature in database
GPG Key ID: 3116BA5E9FFA69A3
1 changed files with 12 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
lib/wayback_machine_downloader/archive_api.rb
View File

@ -1,28 +1,29 @@
require 'uri'
module ArchiveAPI
def get_raw_list_from_api url, page_index
request_url = "https://web.archive.org/cdx/search/xd?url="
request_url += url
request_url += parameters_for_api page_index
request_url = URI("https://web.archive.org/cdx/search/xd")
params = [["url", url]]
params += parameters_for_api page_index
request_url.query = URI.encode_www_form(params)
URI.open(request_url).read
end
def parameters_for_api page_index
parameters = "&fl=timestamp,original&collapse=digest&gzip=false"
if @all
parameters += ""
else
parameters += "&filter=statuscode:200"
parameters = [["fl", "timestamp,original"], ["collapse", "digest"], ["gzip", "false"]]
if !@all
parameters.push(["filter", "statuscode:200"])
end
if @from_timestamp and @from_timestamp != 0
parameters += "&from=" + @from_timestamp.to_s
parameters.push(["from", @from_timestamp.to_s])
end
if @to_timestamp and @to_timestamp != 0
parameters += "&to=" + @to_timestamp.to_s
parameters.push(["to", @to_timestamp.to_s])
end
if page_index
parameters += "&page=#{page_index}"
parameters.push(["page", page_index])
end
parameters
end