jro_apps/redmine Commit - r5253:6536c53e0973

scm: replace invalid utf-8 sequences in comments instead of stripping on Ruby 1.8....

Toshi MARUYAMA -

r5253:6536c53e0973

parent child

Context file:

r5253:6536c53e0973

Collapse all files

app/models/changeset.rb +11 -12

                   str.force_encoding("UTF-8") if str.respond_to?(:force_encoding)
                   return str
                 end
+                enc = encoding.blank? ? "UTF-8" : encoding
                 if str.respond_to?(:force_encoding)
-                  enc = encoding.blank? ? "UTF-8" : encoding
                   if enc != "UTF-8"
                     str.force_encoding(enc)
                     str = str.encode("UTF-8", :invalid => :replace,
                     end
                   end
                 else
-                  unless encoding.blank? || encoding == 'UTF-8'
+                  ic = Iconv.new('UTF-8', enc)
-                    begin
+                  txtar = ""
-                      str = Iconv.conv('UTF-8', encoding, str)
-                    rescue Iconv::Failure
-                      # do nothing here
-                    end
-                  end
-                  # removes invalid UTF8 sequences
                   begin
-                    str = Iconv.conv('UTF-8//IGNORE', 'UTF-8', str + '  ')[0..-3]
+                    txtar += ic.iconv(str)
-                  rescue Iconv::InvalidEncoding
+                  rescue Iconv::IllegalSequence
-                    # "UTF-8//IGNORE" is not supported on some OS
+                    txtar += $!.success
+                    str = '?' + $!.failed[1,$!.failed.length]
+                    retry
+                  rescue
+                    txtar += $!.success
                   end
+                  str = txtar
                 end
                 str
               end

test/unit/changeset_test.rb +13 -19

             class ChangesetTest < ActiveSupport::TestCase
               fixtures :projects, :repositories, :issues, :issue_statuses,
-                       :changesets, :changes, :issue_categories, :enumerations, :custom_fields, :custom_values, :users, :members, :member_roles, :trackers
+                       :changesets, :changes, :issue_categories, :enumerations,
+                       :custom_fields, :custom_values, :users, :members, :member_roles, :trackers
               def setup
               end
                   assert_equal str_utf8, c.comments
               end
-              def test_invalid_utf8_sequences_in_comments_should_be_stripped
+              def test_invalid_utf8_sequences_in_comments_should_be_replaced_latin1
                   proj = Project.find(3)
                   # str = File.read("#{RAILS_ROOT}/test/fixtures/encoding/iso-8859-1.txt")
                   str = "Texte encod\xe9 en ISO-8859-1."
                   str.force_encoding("ASCII-8BIT") if str.respond_to?(:force_encoding)
                   r = Repository::Bazaar.create!(
-                        :project => proj, :url => '/tmp/test/bazaar',
+                        :project => proj,
+                        :url => '/tmp/test/bazaar',
                         :log_encoding => 'UTF-8' )
                   assert r
-                  c = Changeset.new(:repository => r,
+                  c = Changeset.new(:repository   => r,
                                     :committed_on => Time.now,
-                                    :revision => '123',
+                                    :revision     => '123',
-                                    :scmid => '12345',
+                                    :scmid        => '12345',
-                                    :comments => str)
+                                    :comments     => str)
                   assert( c.save )
-                  if str.respond_to?(:force_encoding)
+                  assert_equal "Texte encod? en ISO-8859-1.", c.comments
-                    assert_equal "Texte encod? en ISO-8859-1.", c.comments
-                  else
-                    assert_equal "Texte encod en ISO-8859-1.", c.comments
-                  end
               end
-              def test_invalid_utf8_sequences_in_comments_should_be_stripped_ja_jis
+              def test_invalid_utf8_sequences_in_comments_should_be_replaced_ja_jis
                   proj = Project.find(3)
                   str = "test\xb5\xfetest\xb5\xfe"
                   if str.respond_to?(:force_encoding)
                   end
                   r = Repository::Bazaar.create!(
                         :project => proj,
-                        :url => '/tmp/test/bazaar',
+                        :url     => '/tmp/test/bazaar',
                         :log_encoding => 'ISO-2022-JP' )
                   assert r
                   c = Changeset.new(:repository   => r,
                                     :scmid        => '12345',
                                     :comments     => str)
                   assert( c.save )
-                  if str.respond_to?(:force_encoding)
+                  assert_equal "test??test??", c.comments
-                    assert_equal "test??test??", c.comments
-                  else
-                    assert_equal "testtest", c.comments
-                  end
               end
               def test_comments_should_be_converted_all_latin1_to_utf8

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages