Use a FixedThreadPool for concurrent API calls

lib/wayback_machine_downloader.rb

@@ -289,7 +289,8 @@ class WaybackMachineDownloader
       page_index = 0
       batch_size = [@threads_count, 5].min
       continue_fetching = true
-
+      fetch_pool = Concurrent::FixedThreadPool.new([@threads_count, 1].max)
+      begin
         while continue_fetching && page_index < @maximum_pages
           # Determine the range of pages to fetch in this batch
           end_index = [page_index + batch_size, @maximum_pages].min
@@ -297,7 +298,7 @@ class WaybackMachineDownloader
 
           # Create futures for concurrent API calls
           futures = current_batch.map do |page|
-          Concurrent::Future.execute do
+            Concurrent::Future.execute(executor: fetch_pool) do
               result = nil
               @connection_pool.with_connection do |connection|
                 result = get_raw_list_from_api("#{@base_url}/*", page, connection)
@@ -337,6 +338,10 @@ class WaybackMachineDownloader
 
           sleep(RATE_LIMIT) if continue_fetching
         end
+      ensure
+        fetch_pool.shutdown
+        fetch_pool.wait_for_termination
+      end
     end
 
     puts " found #{snapshot_list_to_consider.length} snapshots."