Fixes for tidy_bytes

admittedly not the cleanest way to do this, although it works for #25.

lib/wayback_machine_downloader/tidy_bytes.rb

@@ -2,24 +2,56 @@
 
 # essentially, this is for converting a string with a potentially
 # broken or unknown encoding into a valid UTF-8 string
+# @todo: consider using charlock_holmes for this in the future
 module TidyBytes
-  def tidy_bytes
+  UNICODE_REPLACEMENT_CHARACTER = "<EFBFBD>"
-    # return if the string is already valid UTF-8
-    return self if self.valid_encoding? && self.encoding == Encoding::UTF_8
 
-    # create a mutable copy so we don't modify the original string
+  # common encodings to try for best multilingual compatibility
-    str = self.dup
+  COMMON_ENCODINGS = [
+    Encoding::UTF_8,
+    Encoding::Windows_1251, # Cyrillic/Russian legacy
+    Encoding::GB18030,      # Simplified Chinese
+    Encoding::Shift_JIS,    # Japanese
+    Encoding::EUC_KR,       # Korean
+    Encoding::ISO_8859_1,   # Western European
+    Encoding::Windows_1252  # Western European/Latin1 superset
+  ].select { |enc| Encoding.name_list.include?(enc.name) }
 
-    # attempt to encode to UTF-8
+  # returns true if the string appears to be binary (has null bytes)
-    begin
+  def binary_data?
-      return str.encode(Encoding::UTF-8)
+    self.include?("\x00".b)
-    rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError
   end
 
-    # if it failed, force the encoding to ISO-8859-1, transcode the
+  # attempts to return a valid UTF-8 version of the string
-    # string to UTF-8, and use replacement options for any characters
+  def tidy_bytes
-    # that might still be problematic
+    return self if self.encoding == Encoding::UTF_8 && self.valid_encoding?
-    str.force_encoding(Encoding::ISO_8859_1).encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: '<27>')
+    return self.dup.force_encoding("BINARY") if binary_data?
+
+    str = self.dup
+    COMMON_ENCODINGS.each do |enc|
+      str.force_encoding(enc)
+      begin
+        utf8 = str.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: UNICODE_REPLACEMENT_CHARACTER)
+        return utf8 if utf8.valid_encoding? && !utf8.include?(UNICODE_REPLACEMENT_CHARACTER)
+      rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError
+        # try next encoding
+      end
+    end
+
+    # if no clean conversion found, try again but accept replacement characters
+    str = self.dup
+    COMMON_ENCODINGS.each do |enc|
+      str.force_encoding(enc)
+      begin
+        utf8 = str.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: UNICODE_REPLACEMENT_CHARACTER)
+        return utf8 if utf8.valid_encoding?
+      rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError
+        # try next encoding
+      end
+    end
+
+    # fallback: replace all invalid/undefined bytes
+    str.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: UNICODE_REPLACEMENT_CHARACTER)
   end
 
   def tidy_bytes!