From b2c0ea2c3e150f8ac666cd1ec0a81b943ee131c8 2017-01-28 06:24:51
From: Toshi MARUYAMA <marutosijp2@yahoo.co.jp>
Date: 2017-01-28 06:24:51
Subject: [PATCH] do not replace all invalid utf8 (#24616)

git-svn-id: http://svn.redmine.org/redmine/trunk@16273 e93f8b46-1217-0410-a6f0-8f06a7374b81

---

diff --git a/lib/redmine/codeset_util.rb b/lib/redmine/codeset_util.rb
index bd4803b..4f056d5 100644
--- a/lib/redmine/codeset_util.rb
+++ b/lib/redmine/codeset_util.rb
@@ -6,7 +6,7 @@ module Redmine
       return str if str.nil?
       str.force_encoding('UTF-8')
       if ! str.valid_encoding?
-        str = str.encode("US-ASCII", :invalid => :replace,
+        str = str.encode("UTF-16LE", :invalid => :replace,
               :undef => :replace, :replace => '?').encode("UTF-8")
       end
       str
diff --git a/test/unit/lib/redmine/codeset_util_test.rb b/test/unit/lib/redmine/codeset_util_test.rb
index 35ab008..dcfc08b 100644
--- a/test/unit/lib/redmine/codeset_util_test.rb
+++ b/test/unit/lib/redmine/codeset_util_test.rb
@@ -91,7 +91,7 @@ class Redmine::CodesetUtilTest < ActiveSupport::TestCase
     s2 = Redmine::CodesetUtil.replace_invalid_utf8(s1)
     assert s2.valid_encoding?
     assert_equal "UTF-8", s2.encoding.to_s
-    assert_equal "??????", s2
+    assert_equal "\xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1??".force_encoding("UTF-8"), s2
   end
 
   test "#to_utf8 should replace invalid non utf8" do