External/wayback-machine-downloader
1
0
Fork 0
mirror of https://github.com/StrawberryMaster/wayback-machine-downloader.git synced 2025-12-17 17:56:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #8 from adampweb/master

Browse Source 
Fix: delete empty files, Compose command fixes
This commit is contained in:
Felipe 2025-05-12 10:36:10 -03:00 committed by GitHub
commit 9709834e20
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 6 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
docker-compose.yml
View File

@ -6,10 +6,10 @@ services:
image: wayback_machine_downloader:latest image: wayback_machine_downloader:latest
container_name: wayback_machine_downloader container_name: wayback_machine_downloader
environment: environment:
- DEVELOPMENT=${DEVELOPMENT:-production} - ENVIRONMENT=${DEVELOPMENT:-production}
- OPTIONS=${OPTIONS:-""} - OPTIONS=${OPTIONS:-""}
- TARGET_URL=${TARGET_URL} - TARGET_URL=${TARGET_URL}
volumes: volumes:
- .:/build:rw - .:/build:rw
- ./websites:/build/websites:rw - ./websites:/websites:rw
command: --directory /build/websites ${OPTIONS} ${TARGET_URL} command: /build/bin/wayback_machine_downloader ${TARGET_URL} ${OPTIONS}

2
lib/wayback_machine_downloader.rb
View File

@ -567,7 +567,7 @@ class WaybackMachineDownloader
"#{file_url} -> #{file_path} (#{@processed_file_count + 1}/#{@total_to_download})" "#{file_url} -> #{file_path} (#{@processed_file_count + 1}/#{@total_to_download})"
rescue StandardError => e rescue StandardError => e
msg = "Failed: #{file_url} # #{e} (#{@processed_file_count + 1}/#{@total_to_download})" msg = "Failed: #{file_url} # #{e} (#{@processed_file_count + 1}/#{@total_to_download})"
if not @all and File.exist?(file_path) and File.size(file_path) == 0 if File.exist?(file_path) and File.size(file_path) == 0
File.delete(file_path) File.delete(file_path)
msg += "\n#{file_path} was empty and was removed." msg += "\n#{file_path} was empty and was removed."
end end

6
wayback_machine_downloader.gemspec
View File

@ -1,8 +1,6 @@
require './lib/wayback_machine_downloader'
Gem::Specification.new do |s| Gem::Specification.new do |s|
s.name = "wayback_machine_downloader_straw" s.name = "wayback_machine_downloader_straw"
s.version = WaybackMachineDownloader::VERSION s.version = "2.3.5"
s.executables << "wayback_machine_downloader" s.executables << "wayback_machine_downloader"
s.summary = "Download an entire website from the Wayback Machine." s.summary = "Download an entire website from the Wayback Machine."
s.description = "Download complete websites from the Internet Archive's Wayback Machine. While the Wayback Machine (archive.org) excellently preserves web history, it lacks a built-in export functionality; this gem does just that, allowing you to download entire archived websites. (This is a significant rewrite of the original wayback_machine_downloader gem by hartator, with enhanced features and performance improvements.)" s.description = "Download complete websites from the Internet Archive's Wayback Machine. While the Wayback Machine (archive.org) excellently preserves web history, it lacks a built-in export functionality; this gem does just that, allowing you to download entire archived websites. (This is a significant rewrite of the original wayback_machine_downloader gem by hartator, with enhanced features and performance improvements.)"
@ -11,7 +9,7 @@ Gem::Specification.new do |s|
s.files = ["lib/wayback_machine_downloader.rb", "lib/wayback_machine_downloader/tidy_bytes.rb", "lib/wayback_machine_downloader/to_regex.rb", "lib/wayback_machine_downloader/archive_api.rb"] s.files = ["lib/wayback_machine_downloader.rb", "lib/wayback_machine_downloader/tidy_bytes.rb", "lib/wayback_machine_downloader/to_regex.rb", "lib/wayback_machine_downloader/archive_api.rb"]
s.homepage = "https://github.com/StrawberryMaster/wayback-machine-downloader" s.homepage = "https://github.com/StrawberryMaster/wayback-machine-downloader"
s.license = "MIT" s.license = "MIT"
s.required_ruby_version = ">= 1.9.2" s.required_ruby_version = ">= 3.4.3"
s.add_runtime_dependency "concurrent-ruby", "~> 1.3", ">= 1.3.4" s.add_runtime_dependency "concurrent-ruby", "~> 1.3", ">= 1.3.4"
s.add_development_dependency "rake", "~> 12.2" s.add_development_dependency "rake", "~> 12.2"
s.add_development_dependency "minitest", "~> 5.2" s.add_development_dependency "minitest", "~> 5.2"