External/wayback-machine-downloader
1
0
Fork 0
mirror of https://github.com/StrawberryMaster/wayback-machine-downloader.git synced 2025-12-17 17:56:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use a FixedThreadPool for concurrent API calls

Browse Source
This commit is contained in:
Felipe 2025-09-24 21:05:22 +00:00 committed by GitHub
parent b1974a8dfa
commit d7a63361e3
1 changed files with 48 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
lib/wayback_machine_downloader.rb
View File

@ -289,7 +289,8 @@ class WaybackMachineDownloader
page_index = 0 page_index = 0
batch_size = [@threads_count, 5].min batch_size = [@threads_count, 5].min
continue_fetching = true continue_fetching = true
fetch_pool = Concurrent::FixedThreadPool.new([@threads_count, 1].max)
begin
while continue_fetching && page_index < @maximum_pages while continue_fetching && page_index < @maximum_pages
# Determine the range of pages to fetch in this batch # Determine the range of pages to fetch in this batch
end_index = [page_index + batch_size, @maximum_pages].min end_index = [page_index + batch_size, @maximum_pages].min
@ -297,7 +298,7 @@ class WaybackMachineDownloader
# Create futures for concurrent API calls # Create futures for concurrent API calls
futures = current_batch.map do |page| futures = current_batch.map do |page|
Concurrent::Future.execute do Concurrent::Future.execute(executor: fetch_pool) do
result = nil result = nil
@connection_pool.with_connection do |connection| @connection_pool.with_connection do |connection|
result = get_raw_list_from_api("#{@base_url}/*", page, connection) result = get_raw_list_from_api("#{@base_url}/*", page, connection)
@ -337,6 +338,10 @@ class WaybackMachineDownloader
sleep(RATE_LIMIT) if continue_fetching sleep(RATE_LIMIT) if continue_fetching
end end
ensure
fetch_pool.shutdown
fetch_pool.wait_for_termination
end
end end
puts " found #{snapshot_list_to_consider.length} snapshots." puts " found #{snapshot_list_to_consider.length} snapshots."