scm: for log in Ruby 1.9, replace invalid UTF-8 to '?' instead of removing.

Refer r3466 #4773. git-svn-id: svn+ssh://rubyforge.org/var/svn/redmine/trunk@4926 e93f8b46-1217-0410-a6f0-8f06a7374b81

scm: for log in Ruby 1.9, replace invalid UTF-8 to '?' instead of removing.
Refer r3466 #4773. git-svn-id: svn+ssh://rubyforge.org/var/svn/redmine/trunk@4926 e93f8b46-1217-0410-a6f0-8f06a7374b81
a81049d5 · Toshi MARUYAMA · Eric Davis · 0e342992 · a81049d5 · a81049d5
Commit a81049d5 authored Feb 22, 2011 by Toshi MARUYAMA Committed by Eric Davis Apr 16, 2011
Hide whitespace changes
Inline Side-by-side

Showing with 21 additions and 8 deletions

changeset.rb app/models/changeset.rb +14 -6

changeset_test.rb test/unit/changeset_test.rb +7 -2

No files found.
--- a/app/models/changeset.rb
+++ b/app/models/changeset.rb
@@ -253,12 +253,20 @@ class Changeset < ActiveRecord::Base
        # do nothing here
      end
    end
-    # removes invalid UTF8 sequences
+    if str.respond_to?(:force_encoding)
-    begin
+      str.force_encoding('UTF-8')
-      Iconv.conv('UTF-8//IGNORE', 'UTF-8', str + '  ')[0..-3]
+      if ! str.valid_encoding?
-    rescue Iconv::InvalidEncoding
+        str = str.encode("US-ASCII", :invalid => :replace,
-      # "UTF-8//IGNORE" is not supported on some OS
+              :undef => :replace, :replace => '?').encode("UTF-8")
-      str
+      end
+    else
+      # removes invalid UTF8 sequences
+      begin
+        str = Iconv.conv('UTF-8//IGNORE', 'UTF-8', str + '  ')[0..-3]
+      rescue Iconv::InvalidEncoding
+        # "UTF-8//IGNORE" is not supported on some OS
+      end
    end
+    str
  end
 end
--- a/test/unit/changeset_test.rb
+++ b/test/unit/changeset_test.rb
@@ -216,8 +216,13 @@ class ChangesetTest < ActiveSupport::TestCase
  def test_invalid_utf8_sequences_in_comments_should_be_stripped
    with_settings :commit_logs_encoding => 'UTF-8' do
      c = Changeset.new
-      c.comments = File.read("#{RAILS_ROOT}/test/fixtures/encoding/iso-8859-1.txt")
+      str = File.read("#{RAILS_ROOT}/test/fixtures/encoding/iso-8859-1.txt")
-      assert_equal "Texte encod en ISO-8859-1.", c.comments
+      c.comments = str
+      if str.respond_to?(:force_encoding)
+        assert_equal "Texte encod? en ISO-8859-1.", c.comments
+      else
+        assert_equal "Texte encod en ISO-8859-1.", c.comments
+      end
    end
  end